From a2216510b772c4145d1142a0372b0072a9b2291e Mon Sep 17 00:00:00 2001 From: dumbmoron <136796770+dumbmoron@users.noreply.github.com> Date: Wed, 16 Aug 2023 19:49:55 +0000 Subject: [PATCH] add cookie support usage: - create cookies.json file somewhere, preferrably outside cobalt directory - in docker, you can bind mount it (`volumes` in composefile) - if you don't want cobalt to update the cookies, set it to `:ro` (cobalt will print a warning about this, ignore it) - set COOKIE_PATH to the absolute path of this file - enjoy? usage in services: probably the simplest api ever - import { getCookie, updateCookie } from '../../cookie/manager.js'; - const cookie = getCookie(''); - add this to headers - `headers: { cookie }` - after fetch is done, save potential cookie updates: updateCookie(cookie, fetch.headers) - see instagram.js for example usage --- .gitignore | 3 + package.json | 1 + src/modules/cookie/cookie.js | 43 +++++++++++ src/modules/cookie/cookies.json.example | 9 +++ src/modules/cookie/manager.js | 78 ++++++++++++++++++++ src/modules/processing/services/instagram.js | 7 +- 6 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 src/modules/cookie/cookie.js create mode 100644 src/modules/cookie/cookies.json.example create mode 100644 src/modules/cookie/manager.js diff --git a/.gitignore b/.gitignore index 7d12aa29..887344cc 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ docker-compose.yml # vscode .vscode + +# cookie file +cookies.json diff --git a/package.json b/package.json index e43894a3..9b8ad056 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "got": "^12.1.0", "nanoid": "^4.0.2", "node-cache": "^5.1.2", + "set-cookie-parser": "2.6.0", "url-pattern": "1.0.3", "xml-js": "^1.6.11", "youtubei.js": "^5.4.0" diff --git a/src/modules/cookie/cookie.js b/src/modules/cookie/cookie.js new file mode 100644 index 00000000..01a07de5 --- /dev/null +++ b/src/modules/cookie/cookie.js @@ -0,0 +1,43 @@ +import { strict as assert } from 'node:assert' + +export default class Cookie { + constructor(input) { + assert(typeof input === 'object'); + this._values = {} + this.set(input); + } + + set(values) { + Object.entries(values).forEach( + ([ key, value ]) => this._values[key] = value + ) + } + + unset(keys) { + for (const key of keys) + delete this._values[key] + } + + static fromString(str) { + const obj = {} + + str + .split('; ') + .forEach(cookie => { + const key = cookie.split('=')[0] + const value = cookie.split('=').splice(1).join('=') + obj[key] = decodeURIComponent(value) + }) + + return new Cookie(obj); + } + + toString() { + return Object.entries(this._values) + .map(([ name, value ]) => `${name}=${encodeURIComponent(value)}`) + .join('; '); + } + + toJSON() { return this.toString() } + values() { return Object.freeze({ ...this._values }) } +} \ No newline at end of file diff --git a/src/modules/cookie/cookies.json.example b/src/modules/cookie/cookies.json.example new file mode 100644 index 00000000..60cab650 --- /dev/null +++ b/src/modules/cookie/cookies.json.example @@ -0,0 +1,9 @@ +{ + "instagram": [ + "cookie=asd; bla=bla; fake=cookie" + ], + "youtube": [ + "epic=google_cookie", + "epic=another_epic; youtube=cookie" + ] +} \ No newline at end of file diff --git a/src/modules/cookie/manager.js b/src/modules/cookie/manager.js new file mode 100644 index 00000000..4e29151a --- /dev/null +++ b/src/modules/cookie/manager.js @@ -0,0 +1,78 @@ +import path from 'path' +import Cookie from './cookie.js' +import { readFile, writeFile } from 'fs/promises' +import { parse as parseSetCookie, splitCookiesString } from 'set-cookie-parser' + +const WRITE_INTERVAL = 60000, + COOKIE_PATH = process.env.COOKIE_PATH, + COUNTER = Symbol('counter'); + +let cookies = {}, dirty = false, intervalId; + +const setup = async () => { + try { + if (!COOKIE_PATH) + return + + cookies = await readFile(COOKIE_PATH, 'utf8') + cookies = JSON.parse(cookies) + intervalId = setInterval(writeChanges, WRITE_INTERVAL) + } catch { /* no cookies for you */ } +} + +setup() + +function writeChanges() { + if (!dirty) return + dirty = false + + writeFile( + COOKIE_PATH, + JSON.stringify(cookies, null, 4) + ).catch(e => { + console.error('warn: cookies failed to save, stopping interval') + console.error('exception:', e) + clearInterval(intervalId) + }) +} + +export function getCookie(service) { + if (!cookies[service] || !cookies[service].length) + return + + let n + if (cookies[service][COUNTER] === undefined) { + n = cookies[service][COUNTER] = 0 + } else { + ++cookies[service][COUNTER] + n = (cookies[service][COUNTER] %= cookies[service].length) + } + + const cookie = cookies[service][n] + if (typeof cookie === 'string') + cookies[service][n] = Cookie.fromString(cookie) + + return cookies[service][n] +} + +// todo: expiry checking? domain checking? +// might be pointless for the purposes of cobalt +export function updateCookie(cookie, headers) { + const parsed = parseSetCookie( + splitCookiesString(headers.get('set-cookie')) + ), values = {} + + cookie.unset( + parsed + .filter(c => c.expires < new Date()) + .map(c => c.name) + ) + + parsed + .filter(c => c.expires > new Date()) + .forEach(c => values[c.name] = c.value); + + cookie.set(values) + if (Object.keys(values).length) + dirty = true +} diff --git a/src/modules/processing/services/instagram.js b/src/modules/processing/services/instagram.js index d1714393..b45af479 100644 --- a/src/modules/processing/services/instagram.js +++ b/src/modules/processing/services/instagram.js @@ -1,5 +1,6 @@ import { createStream } from "../../stream/manage.js"; import { genericUserAgent } from "../../config.js"; +import { getCookie, updateCookie } from '../../cookie/manager.js'; export default async function(obj) { let data; @@ -14,6 +15,8 @@ export default async function(obj) { shortcode: obj.id })) + const cookie = getCookie('instagram'); + data = await fetch(url, { headers: { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', @@ -25,9 +28,11 @@ export default async function(obj) { 'Sec-Fetch-Site': 'same-origin', 'upgrade-insecure-requests': '1', 'accept-encoding': 'gzip, deflate, br', - 'accept-language': 'en-US,en;q=0.9,en;q=0.8' + 'accept-language': 'en-US,en;q=0.9,en;q=0.8', + cookie } }) + updateCookie(cookie, data.headers); data = (await data.json()).data; } catch (e) { data = false;