mirror of
https://github.com/imputnet/cobalt.git
synced 2025-02-13 17:00:12 +00:00
Merge remote-tracking branch 'upstream/main' into feat/threads
This commit is contained in:
commit
16f4442dc2
.github
CONTRIBUTING.mdDockerfileREADME.mdapi
README.mdpackage.json
src
cobalt.jsconfig.js
core
misc
processing
cookie
create-filename.jsmatch-action.jsmatch.jsrequest.jsschema.jsservice-config.jsservice-patterns.jsservices
bluesky.jsdailymotion.jsinstagram.jsok.jsrutube.jssnapchat.jssoundcloud.jstiktok.jstumblr.jstwitch.jstwitter.jsvimeo.jsvine.jsvk.jsxiaohongshu.jsyoutube.js
url.jssecurity
store
stream
util
docs
api.mdconfigure-for-youtube.mdprotect-an-instance.mdrun-an-instance.mdtroubleshooting.md
examples
images
protect-an-instance
troubleshooting/clipboard
3
.github/test.sh
vendored
3
.github/test.sh
vendored
|
@ -18,7 +18,7 @@ test_api() {
|
|||
-X POST \
|
||||
-H "Accept: application/json" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"url":"https://www.tiktok.com/@fatfatmillycat/video/7195741644585454894"}')
|
||||
-d '{"url":"https://garfield-69.tumblr.com/post/696499862852780032","alwaysProxy":true}')
|
||||
|
||||
echo "API_RESPONSE=$API_RESPONSE"
|
||||
STATUS=$(echo "$API_RESPONSE" | jq -r .status)
|
||||
|
@ -46,6 +46,7 @@ setup_api() {
|
|||
}
|
||||
|
||||
setup_web() {
|
||||
pnpm run --prefix web check
|
||||
pnpm run --prefix web build
|
||||
}
|
||||
|
||||
|
|
93
.github/workflows/codeql.yml
vendored
Normal file
93
.github/workflows/codeql.yml
vendored
Normal file
|
@ -0,0 +1,93 @@
|
|||
# For most projects, this workflow file will not need changing; you simply need
|
||||
# to commit it to your repository.
|
||||
#
|
||||
# You may wish to alter this file to override the set of languages analyzed,
|
||||
# or to provide custom queries or build logic.
|
||||
#
|
||||
# ******** NOTE ********
|
||||
# We have attempted to detect the languages in your repository. Please check
|
||||
# the `language` matrix defined below to confirm you have the correct set of
|
||||
# supported CodeQL languages.
|
||||
#
|
||||
name: "CodeQL"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- '**'
|
||||
pull_request:
|
||||
branches: [ "main", "7" ]
|
||||
schedule:
|
||||
- cron: '33 7 * * 5'
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze (${{ matrix.language }})
|
||||
# Runner size impacts CodeQL analysis time. To learn more, please see:
|
||||
# - https://gh.io/recommended-hardware-resources-for-running-codeql
|
||||
# - https://gh.io/supported-runners-and-hardware-resources
|
||||
# - https://gh.io/using-larger-runners (GitHub.com only)
|
||||
# Consider using larger runners or machines with greater resources for possible analysis time improvements.
|
||||
runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
# required for all workflows
|
||||
security-events: write
|
||||
|
||||
# required to fetch internal or private CodeQL packs
|
||||
packages: read
|
||||
|
||||
# only required for workflows in private repositories
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- language: javascript-typescript
|
||||
build-mode: none
|
||||
# CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
|
||||
# Use `c-cpp` to analyze code written in C, C++ or both
|
||||
# Use 'java-kotlin' to analyze code written in Java, Kotlin or both
|
||||
# Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
|
||||
# To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
|
||||
# see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
|
||||
# If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
|
||||
# your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v3
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
build-mode: ${{ matrix.build-mode }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
# By default, queries listed here will override any specified in a config file.
|
||||
# Prefix the list here with "+" to use these queries and those in the config file.
|
||||
|
||||
# For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
|
||||
# queries: security-extended,security-and-quality
|
||||
|
||||
# If the analyze step fails for one of the languages you are analyzing with
|
||||
# "We were unable to automatically build your code", modify the matrix above
|
||||
# to set the build mode to "manual" for that language. Then modify this step
|
||||
# to build your code.
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
|
||||
- if: matrix.build-mode == 'manual'
|
||||
shell: bash
|
||||
run: |
|
||||
echo 'If you are using a "manual" build mode for one or more of the' \
|
||||
'languages you are analyzing, replace this with the commands to build' \
|
||||
'your code, for example:'
|
||||
echo ' make bootstrap'
|
||||
echo ' make release'
|
||||
exit 1
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v3
|
||||
with:
|
||||
category: "/language:${{matrix.language}}"
|
2
.github/workflows/docker.yml
vendored
2
.github/workflows/docker.yml
vendored
|
@ -51,7 +51,7 @@ jobs:
|
|||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64,linux/arm/v7
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
|
|
4
.github/workflows/test-services.yml
vendored
4
.github/workflows/test-services.yml
vendored
|
@ -17,7 +17,7 @@ jobs:
|
|||
- uses: actions/checkout@v4
|
||||
- uses: pnpm/action-setup@v4
|
||||
- id: checkServices
|
||||
run: pnpm i --frozen-lockfile && echo "service_list=$(node api/src/util/test-ci get-services)" >> "$GITHUB_OUTPUT"
|
||||
run: pnpm i --frozen-lockfile && echo "service_list=$(node api/src/util/test get-services)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
test-services:
|
||||
needs: check-services
|
||||
|
@ -30,4 +30,4 @@ jobs:
|
|||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: pnpm/action-setup@v4
|
||||
- run: pnpm i --frozen-lockfile && node api/src/util/test-ci run-tests-for ${{ matrix.service }}
|
||||
- run: pnpm i --frozen-lockfile && node api/src/util/test run-tests-for ${{ matrix.service }}
|
||||
|
|
|
@ -4,7 +4,23 @@ if you're reading this, you are probably interested in contributing to cobalt, w
|
|||
this document serves as a guide to help you make contributions that we can merge into the cobalt codebase.
|
||||
|
||||
## translations
|
||||
currently, we are **not accepting** translations of cobalt. this is because we are making significant changes to the frontend, and the currently used localization structure is being completely reworked. if this changes, this document will be updated.
|
||||
we are currently accepting translations via the [i18n platform](https://i18n.imput.net).
|
||||
|
||||
thank you for showing interest in making cobalt more accessible around the world, we really appreciate it! here are some guidelines for how a cobalt translation should look:
|
||||
|
||||
- cobalt's writing style is informal. please do not use formal language, unless there is no other way to express the same idea of the original text in your language.
|
||||
- all cobalt text is written in lowercase. this is a stylistic choice, please do not capitalize translated sentences.
|
||||
- do not translate the name "cobalt", or "imput"
|
||||
- you can translate "meowbalt" into whatever your language's equivalent of _meow_ is (e.g. _miaubalt_ in German)
|
||||
- **please don't translate cobalt into languages which you are not experienced in.** we can use google translate ourselves, but we would prefer cobalt to be translated by humans, not computers.
|
||||
|
||||
if your language does not exist on the translation platform yet, you can request to add it by adding it to any of cobalt's components (e.g. [here](https://i18n.imput.net/projects/cobalt/about/)).
|
||||
|
||||
before translating a piece of text, check that no one has made a translation yet. pending translations are displayed in the **Suggestions** tab on the translate page. if someone already made a suggestion, and you think it's correct, you can upvote it! this helps us distinguish that a translation is correct.
|
||||
|
||||
if no one has submitted a translation, or the submitted translation seems wrong to you, you can submit your translation by clicking the **Suggest** button for each individual string, which sends it off for human review. we will then check it to to ensure no malicious translations are submitted, and add it to cobalt.
|
||||
|
||||
if any translation string's meaning seems unclear to you, please leave a comment on the *Comments* tab, and we will either add an explanation or a screenshot.
|
||||
|
||||
## adding features or support for services
|
||||
before putting in the effort to implement a feature, it's worth considering whether it would be appropriate to add it to cobalt. the cobalt api is built to assist people **only with downloading freely accessible content**. other functionality, such as:
|
||||
|
@ -22,9 +38,9 @@ when contributing code to cobalt, there are a few guidelines in place to ensure
|
|||
### clean commit messages
|
||||
internally, we use a format similar to [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) - the first part signifies which part of the code you are changing (the *scope*), and the second part explains the change. for inspiration on how to write appropriate commit titles, you can take a look at the [commit history](https://github.com/imputnet/cobalt/commits/).
|
||||
|
||||
the scope is not strictly defined, you can write whatever you find most fitting for the particular change. suppose you are changing a small part of a more significant part of the codebase. in that case, you can specify both the larger and smaller scopes in the commit message for clarity (e.g., if you were changing something in internal streams, the commit could be something like `stream/internal: fix object not being handled properly`).
|
||||
the scope is not strictly defined, you can write whatever you find most fitting for the particular change. suppose you are changing a small part of a more significant part of the codebase. in that case, you can specify both the larger and smaller scopes in the commit message for clarity (e.g., if you were changing something in internal streams, the commit could be something like `api/stream: fix object not being handled properly`).
|
||||
|
||||
if you think a change deserves further explanation, we encourage you to write a short explanation in the commit message ([example](https://github.com/imputnet/cobalt/commit/d2e5b6542f71f3809ba94d56c26f382b5cb62762)), which will save both you and us time having to enquire about the change, and you explaining the reason behind it.
|
||||
if you think a change deserves further explanation, we encourage you to write a short explanation in the commit message ([example](https://github.com/imputnet/cobalt/commit/31be60484de8eaf63bba8a4f508e16438aa7ba6e)), which will save both you and us time having to enquire about the change, and you explaining the reason behind it.
|
||||
|
||||
if your contribution has uninformative commit titles, you may be asked to interactively rebase your branch and amend each commit to include a meaningful title.
|
||||
|
||||
|
|
13
Dockerfile
13
Dockerfile
|
@ -1,4 +1,4 @@
|
|||
FROM node:20-bullseye-slim AS base
|
||||
FROM node:23-alpine AS base
|
||||
ENV PNPM_HOME="/pnpm"
|
||||
ENV PATH="$PNPM_HOME:$PATH"
|
||||
|
||||
|
@ -7,8 +7,7 @@ WORKDIR /app
|
|||
COPY . /app
|
||||
|
||||
RUN corepack enable
|
||||
RUN apt-get update && \
|
||||
apt-get install -y python3 build-essential
|
||||
RUN apk add --no-cache python3 alpine-sdk
|
||||
|
||||
RUN --mount=type=cache,id=pnpm,target=/pnpm/store \
|
||||
pnpm install --prod --frozen-lockfile
|
||||
|
@ -18,8 +17,10 @@ RUN pnpm deploy --filter=@imput/cobalt-api --prod /prod/api
|
|||
FROM base AS api
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=build /prod/api /app
|
||||
COPY --from=build /app/.git /app/.git
|
||||
COPY --from=build --chown=node:node /prod/api /app
|
||||
COPY --from=build --chown=node:node /app/.git /app/.git
|
||||
|
||||
USER node
|
||||
|
||||
EXPOSE 9000
|
||||
CMD [ "node", "src/cobalt" ]
|
||||
CMD [ "node", "src/cobalt" ]
|
||||
|
|
120
README.md
120
README.md
|
@ -14,111 +14,47 @@
|
|||
<a href="https://discord.gg/pQPt8HBUPu">
|
||||
💬 community discord server
|
||||
</a>
|
||||
<br/>
|
||||
<a href="https://x.com/justusecobalt">
|
||||
🐦 twitter/x
|
||||
🐦 twitter
|
||||
</a>
|
||||
<a href="https://bsky.app/profile/cobalt.tools">
|
||||
🦋 bluesky
|
||||
</a>
|
||||
</p>
|
||||
<br/>
|
||||
</div>
|
||||
|
||||
cobalt is a media downloader that doesn't piss you off. it's fast, friendly, and doesn't have any bullshit that modern web is filled with: ***no ads, trackers, or paywalls***.
|
||||
cobalt is a media downloader that doesn't piss you off. it's friendly, efficient, and doesn't have ads, trackers, paywalls or other nonsense.
|
||||
|
||||
paste the link, get the file, move on. it's that simple. just how it should be.
|
||||
paste the link, get the file, move on. that simple, just how it should be.
|
||||
|
||||
### supported services
|
||||
this list is not final and keeps expanding over time. if support for a service you want is missing, create an issue (or a pull request 👀).
|
||||
### cobalt monorepo
|
||||
this monorepo includes source code for api, frontend, and related packages:
|
||||
- [api tree & readme](/api/)
|
||||
- [web tree & readme](/web/)
|
||||
- [packages tree](/packages/)
|
||||
|
||||
| service | video + audio | only audio | only video | metadata | rich file names |
|
||||
| :-------- | :-----------: | :--------: | :--------: | :------: | :-------------: |
|
||||
| bilibili | ✅ | ✅ | ✅ | ➖ | ➖ |
|
||||
| bluesky | ✅ | ✅ | ✅ | ➖ | ➖ |
|
||||
| dailymotion | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| instagram | ✅ | ✅ | ✅ | ➖ | ➖ |
|
||||
| facebook | ✅ | ❌ | ✅ | ➖ | ➖ |
|
||||
| loom | ✅ | ❌ | ✅ | ✅ | ➖ |
|
||||
| ok.ru | ✅ | ❌ | ✅ | ✅ | ✅ |
|
||||
| pinterest | ✅ | ✅ | ✅ | ➖ | ➖ |
|
||||
| reddit | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| rutube | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| snapchat | ✅ | ✅ | ✅ | ➖ | ➖ |
|
||||
| soundcloud | ➖ | ✅ | ➖ | ✅ | ✅ |
|
||||
| streamable | ✅ | ✅ | ✅ | ➖ | ➖ |
|
||||
| threads posts | ✅ | ✅ | ✅ | ➖ | ➖ |
|
||||
| tiktok | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| tumblr | ✅ | ✅ | ✅ | ➖ | ➖ |
|
||||
| twitch clips | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| twitter/x | ✅ | ✅ | ✅ | ➖ | ➖ |
|
||||
| vimeo | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| vine | ✅ | ✅ | ✅ | ➖ | ➖ |
|
||||
| vk videos & clips | ✅ | ❌ | ✅ | ✅ | ✅ |
|
||||
| youtube | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
it also includes documentation in the [docs tree](/docs/):
|
||||
- [cobalt api documentation](/docs/api.md)
|
||||
- [how to run a cobalt instance](/docs/run-an-instance.md)
|
||||
- [how to protect a cobalt instance](/docs/protect-an-instance.md) (recommended if you host a public instance)
|
||||
|
||||
| emoji | meaning |
|
||||
| :-----: | :---------------------- |
|
||||
| ✅ | supported |
|
||||
| ➖ | impossible/unreasonable |
|
||||
| ❌ | not supported |
|
||||
### thank you
|
||||
cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt). a part of our infrastructure is hosted on their network. we really appreciate their kindness and support!
|
||||
|
||||
### additional notes or features (per service)
|
||||
| service | notes or features |
|
||||
| :-------- | :----- |
|
||||
| instagram | supports reels, photos, and videos. lets you pick what to save from multi-media posts. |
|
||||
| facebook | supports public accessible videos content only. |
|
||||
| pinterest | supports photos, gifs, videos and stories. |
|
||||
| reddit | supports gifs and videos. |
|
||||
| snapchat | supports spotlights and stories. lets you pick what to save from stories. |
|
||||
| rutube | supports yappy & private links. |
|
||||
| soundcloud | supports private links. |
|
||||
| threads | supports photos and videos. lets you pick what to save from multi-media posts. |
|
||||
| tiktok | supports videos with or without watermark, images from slideshow without watermark, and full (original) audios. |
|
||||
| twitter/x | lets you pick what to save from multi-media posts. may not be 100% reliable due to current management. |
|
||||
| vimeo | audio downloads are only available for dash. |
|
||||
| youtube | supports videos, music, and shorts. 8K, 4K, HDR, VR, and high FPS videos. rich metadata & dubs. h264/av1/vp9 codecs. |
|
||||
### ethics
|
||||
cobalt is a tool that makes downloading public content easier. it takes **zero liability**.
|
||||
the end user is responsible for what they download, how they use and distribute that content.
|
||||
cobalt never caches any content, it [works like a fancy proxy](/api/src/stream/).
|
||||
|
||||
### partners
|
||||
cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt), all main instances are currently hosted on their network :)
|
||||
cobalt is in no way a piracy tool and cannot be used as such.
|
||||
it can only download free & publicly accessible content.
|
||||
same content can be downloaded via dev tools of any modern web browser.
|
||||
|
||||
### ethics and disclaimer
|
||||
cobalt is a tool for easing content downloads from internet and takes ***zero liability***. you are responsible for what you download, how you use and distribute that content. please be mindful when using content of others and always credit original creators. fair use and credits benefit everyone.
|
||||
### contributing
|
||||
if you're considering contributing to cobalt, first of all, thank you! check the [contribution guidelines here](/CONTRIBUTING.md) before getting started, they'll help you do your best right away.
|
||||
|
||||
cobalt is ***NOT*** a piracy tool and cannot be used as such. it can only download free, publicly accessible content. such content can be easily downloaded through any browser's dev tools. pressing one button is easier, so i made a convenient, ad-less tool for such repeated actions.
|
||||
|
||||
### cobalt license
|
||||
### licenses
|
||||
for relevant licensing information, see the [api](api/README.md) and [web](web/README.md) READMEs.
|
||||
unless specified otherwise, the remainder of this repository is licensed under [AGPL-3.0](LICENSE).
|
||||
|
||||
## acknowledgements
|
||||
### ffmpeg
|
||||
cobalt heavily relies on ffmpeg for converting and merging media files. it's an absolutely amazing piece of software offered for anyone for free, yet doesn't receive as much credit as it should.
|
||||
|
||||
you can [support ffmpeg here](https://ffmpeg.org/donations.html)!
|
||||
|
||||
#### ffmpeg-static
|
||||
we use [ffmpeg-static](https://github.com/eugeneware/ffmpeg-static) to get binaries for ffmpeg depending on the platform.
|
||||
|
||||
you can support the developer via various methods listed on their github page! (linked above)
|
||||
|
||||
### youtube.js
|
||||
cobalt relies on [youtube.js](https://github.com/LuanRT/YouTube.js) for interacting with the innertube api, it wouldn't have been possible without it.
|
||||
|
||||
you can support the developer via various methods listed on their github page! (linked above)
|
||||
|
||||
### many others
|
||||
cobalt also depends on:
|
||||
|
||||
- [content-disposition-header](https://www.npmjs.com/package/content-disposition-header) to simplify the provision of `content-disposition` headers.
|
||||
- [cors](https://www.npmjs.com/package/cors) to manage cross-origin resource sharing within expressjs.
|
||||
- [dotenv](https://www.npmjs.com/package/dotenv) to load environment variables from the `.env` file.
|
||||
- [esbuild](https://www.npmjs.com/package/esbuild) to minify the frontend files.
|
||||
- [express](https://www.npmjs.com/package/express) as the backbone of cobalt servers.
|
||||
- [express-rate-limit](https://www.npmjs.com/package/express-rate-limit) to rate limit api endpoints.
|
||||
- [hls-parser](https://www.npmjs.com/package/hls-parser) to parse `m3u8` playlists for certain services.
|
||||
- [ipaddr.js](https://www.npmjs.com/package/ipaddr.js) to parse ip addresses (for rate limiting).
|
||||
- [nanoid](https://www.npmjs.com/package/nanoid) to generate unique (temporary) identifiers for each requested stream.
|
||||
- [node-cache](https://www.npmjs.com/package/node-cache) to cache stream info in server ram for a limited amount of time.
|
||||
- [psl](https://www.npmjs.com/package/psl) as the domain name parser.
|
||||
- [set-cookie-parser](https://www.npmjs.com/package/set-cookie-parser) to parse cookies that cobalt receives from certain services.
|
||||
- [undici](https://www.npmjs.com/package/undici) for making http requests.
|
||||
- [url-pattern](https://www.npmjs.com/package/url-pattern) to match provided links with supported patterns.
|
||||
|
||||
...and many other packages that these packages rely on.
|
||||
|
|
|
@ -1,4 +1,64 @@
|
|||
# cobalt api
|
||||
this directory includes the source code for cobalt api. it's made with [express.js](https://www.npmjs.com/package/express) and love!
|
||||
|
||||
## running your own instance
|
||||
if you want to run your own instance for whatever purpose, [follow this guide](/docs/run-an-instance.md).
|
||||
we recommend to use docker compose unless you intend to run cobalt for developing/debugging purposes.
|
||||
|
||||
## accessing the api
|
||||
there is currently no publicly available pre-hosted api.
|
||||
we recommend [deploying your own instance](/docs/run-an-instance.md) if you wish to use the cobalt api.
|
||||
|
||||
you can read [the api documentation here](/docs/api.md).
|
||||
|
||||
## supported services
|
||||
this list is not final and keeps expanding over time!
|
||||
if the desired service isn't supported yet, feel free to create an appropriate issue (or a pull request 👀).
|
||||
|
||||
| service | video + audio | only audio | only video | metadata | rich file names |
|
||||
| :-------- | :-----------: | :--------: | :--------: | :------: | :-------------: |
|
||||
| bilibili | ✅ | ✅ | ✅ | ➖ | ➖ |
|
||||
| bluesky | ✅ | ✅ | ✅ | ➖ | ➖ |
|
||||
| dailymotion | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| instagram | ✅ | ✅ | ✅ | ➖ | ➖ |
|
||||
| facebook | ✅ | ❌ | ✅ | ➖ | ➖ |
|
||||
| loom | ✅ | ❌ | ✅ | ✅ | ➖ |
|
||||
| ok.ru | ✅ | ❌ | ✅ | ✅ | ✅ |
|
||||
| pinterest | ✅ | ✅ | ✅ | ➖ | ➖ |
|
||||
| reddit | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| rutube | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| snapchat | ✅ | ✅ | ✅ | ➖ | ➖ |
|
||||
| soundcloud | ➖ | ✅ | ➖ | ✅ | ✅ |
|
||||
| streamable | ✅ | ✅ | ✅ | ➖ | ➖ |
|
||||
| tiktok | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| tumblr | ✅ | ✅ | ✅ | ➖ | ➖ |
|
||||
| twitch clips | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| twitter/x | ✅ | ✅ | ✅ | ➖ | ➖ |
|
||||
| vimeo | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| vk videos & clips | ✅ | ❌ | ✅ | ✅ | ✅ |
|
||||
| xiaohongshu | ✅ | ✅ | ✅ | ➖ | ➖ |
|
||||
| youtube | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
|
||||
| emoji | meaning |
|
||||
| :-----: | :---------------------- |
|
||||
| ✅ | supported |
|
||||
| ➖ | unreasonable/impossible |
|
||||
| ❌ | not supported |
|
||||
|
||||
### additional notes or features (per service)
|
||||
| service | notes or features |
|
||||
| :-------- | :----- |
|
||||
| instagram | supports reels, photos, and videos. lets you pick what to save from multi-media posts. |
|
||||
| facebook | supports public accessible videos content only. |
|
||||
| pinterest | supports photos, gifs, videos and stories. |
|
||||
| reddit | supports gifs and videos. |
|
||||
| snapchat | supports spotlights and stories. lets you pick what to save from stories. |
|
||||
| rutube | supports yappy & private links. |
|
||||
| soundcloud | supports private links. |
|
||||
| tiktok | supports videos with or without watermark, images from slideshow without watermark, and full (original) audios. |
|
||||
| twitter/x | lets you pick what to save from multi-media posts. may not be 100% reliable due to current management. |
|
||||
| vimeo | audio downloads are only available for dash. |
|
||||
| youtube | supports videos, music, and shorts. 8K, 4K, HDR, VR, and high FPS videos. rich metadata & dubs. h264/av1/vp9 codecs. |
|
||||
|
||||
## license
|
||||
cobalt api code is licensed under [AGPL-3.0](LICENSE).
|
||||
|
@ -9,14 +69,35 @@ as long as you:
|
|||
- provide a link to the license and indicate if changes to the code were made, and
|
||||
- release the code under the **same license**
|
||||
|
||||
## running your own instance
|
||||
if you want to run your own instance for whatever purpose, [follow this guide](/docs/run-an-instance.md).
|
||||
it's *highly* recommended to use a docker compose method unless you run for developing/debugging purposes.
|
||||
## open source acknowledgements
|
||||
### ffmpeg
|
||||
cobalt relies on ffmpeg for muxing and encoding media files. ffmpeg is absolutely spectacular and we're privileged to have an ability to use it for free, just like anyone else. we believe it should be way more recognized.
|
||||
|
||||
## accessing the api
|
||||
currently, there is no publicly accessible main api. we plan on providing a public api for
|
||||
cobalt 10 in some form in the future. we recommend deploying your own instance if you wish
|
||||
to use the latest api. you can access [the documentation](/docs/api.md) for it here.
|
||||
you can [support ffmpeg here](https://ffmpeg.org/donations.html)!
|
||||
|
||||
if you are looking for the documentation for the old (7.x) api, you can find
|
||||
it [here](https://github.com/imputnet/cobalt/blob/7/docs/api.md)
|
||||
### youtube.js
|
||||
cobalt relies on **[youtube.js](https://github.com/LuanRT/YouTube.js)** for interacting with youtube's innertube api, it wouldn't have been possible without this package.
|
||||
|
||||
you can support the developer via various methods listed on their github page!
|
||||
(linked above)
|
||||
|
||||
### many others
|
||||
cobalt-api also depends on:
|
||||
|
||||
- **[content-disposition-header](https://www.npmjs.com/package/content-disposition-header)** to simplify the provision of `content-disposition` headers.
|
||||
- **[cors](https://www.npmjs.com/package/cors)** to manage cross-origin resource sharing within expressjs.
|
||||
- **[dotenv](https://www.npmjs.com/package/dotenv)** to load environment variables from the `.env` file.
|
||||
- **[express](https://www.npmjs.com/package/express)** as the backbone of cobalt servers.
|
||||
- **[express-rate-limit](https://www.npmjs.com/package/express-rate-limit)** to rate limit api endpoints.
|
||||
- **[ffmpeg-static](https://www.npmjs.com/package/ffmpeg-static)** to get binaries for ffmpeg depending on the platform.
|
||||
- **[hls-parser](https://www.npmjs.com/package/hls-parser)** to parse HLS playlists according to spec (very impressive stuff).
|
||||
- **[ipaddr.js](https://www.npmjs.com/package/ipaddr.js)** to parse ip addresses (used for rate limiting).
|
||||
- **[nanoid](https://www.npmjs.com/package/nanoid)** to generate unique identifiers for each requested tunnel.
|
||||
- **[set-cookie-parser](https://www.npmjs.com/package/set-cookie-parser)** to parse cookies that cobalt receives from certain services.
|
||||
- **[undici](https://www.npmjs.com/package/undici)** for making http requests.
|
||||
- **[url-pattern](https://www.npmjs.com/package/url-pattern)** to match provided links with supported patterns.
|
||||
- **[zod](https://www.npmjs.com/package/zod)** to lock down the api request schema.
|
||||
- **[@datastructures-js/priority-queue](https://www.npmjs.com/package/@datastructures-js/priority-queue)** for sorting stream caches for future clean up (without redis).
|
||||
- **[@imput/psl](https://www.npmjs.com/package/@imput/psl)** as the domain name parser, our fork of [psl](https://www.npmjs.com/package/psl).
|
||||
|
||||
...and many other packages that these packages rely on.
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"name": "@imput/cobalt-api",
|
||||
"description": "save what you love",
|
||||
"version": "10.1.0",
|
||||
"version": "10.6",
|
||||
"author": "imput",
|
||||
"exports": "./src/cobalt.js",
|
||||
"type": "module",
|
||||
|
@ -10,9 +10,9 @@
|
|||
},
|
||||
"scripts": {
|
||||
"start": "node src/cobalt",
|
||||
"setup": "node src/util/setup",
|
||||
"test": "node src/util/test",
|
||||
"token:youtube": "node src/util/generate-youtube-tokens"
|
||||
"token:youtube": "node src/util/generate-youtube-tokens",
|
||||
"token:jwt": "node src/util/generate-jwt-secret"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
|
@ -24,26 +24,27 @@
|
|||
},
|
||||
"homepage": "https://github.com/imputnet/cobalt#readme",
|
||||
"dependencies": {
|
||||
"@datastructures-js/priority-queue": "^6.3.1",
|
||||
"@imput/psl": "^2.0.4",
|
||||
"@imput/version-info": "workspace:^",
|
||||
"content-disposition-header": "0.6.0",
|
||||
"cors": "^2.8.5",
|
||||
"dotenv": "^16.0.1",
|
||||
"esbuild": "^0.14.51",
|
||||
"express": "^4.21.0",
|
||||
"express-rate-limit": "^6.3.0",
|
||||
"express": "^4.21.2",
|
||||
"express-rate-limit": "^7.4.1",
|
||||
"ffmpeg-static": "^5.1.0",
|
||||
"hls-parser": "^0.10.7",
|
||||
"ipaddr.js": "2.2.0",
|
||||
"nanoid": "^4.0.2",
|
||||
"node-cache": "^5.1.2",
|
||||
"psl": "1.9.0",
|
||||
"nanoid": "^5.0.9",
|
||||
"set-cookie-parser": "2.6.0",
|
||||
"undici": "^5.19.1",
|
||||
"url-pattern": "1.0.3",
|
||||
"youtubei.js": "^10.5.0",
|
||||
"youtubei.js": "^13.0.0",
|
||||
"zod": "^3.23.8"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"freebind": "^0.2.2"
|
||||
"freebind": "^0.2.2",
|
||||
"rate-limit-redis": "^4.2.0",
|
||||
"redis": "^4.7.0"
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,27 +1,32 @@
|
|||
import "dotenv/config";
|
||||
|
||||
import express from "express";
|
||||
import cluster from "node:cluster";
|
||||
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import path from "path";
|
||||
import { fileURLToPath } from "url";
|
||||
|
||||
import { env } from "./config.js"
|
||||
import { Bright, Green, Red } from "./misc/console-text.js";
|
||||
import { env, isCluster } from "./config.js"
|
||||
import { Red } from "./misc/console-text.js";
|
||||
import { initCluster } from "./misc/cluster.js";
|
||||
|
||||
const app = express();
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename).slice(0, -4);
|
||||
|
||||
app.disable('x-powered-by');
|
||||
app.disable("x-powered-by");
|
||||
|
||||
if (env.apiURL) {
|
||||
const { runAPI } = await import('./core/api.js');
|
||||
runAPI(express, app, __dirname)
|
||||
const { runAPI } = await import("./core/api.js");
|
||||
|
||||
if (isCluster) {
|
||||
await initCluster();
|
||||
}
|
||||
|
||||
runAPI(express, app, __dirname, cluster.isPrimary);
|
||||
} else {
|
||||
console.log(
|
||||
Red(`cobalt wasn't configured yet or configuration is invalid.\n`)
|
||||
+ Bright(`please run the setup script to fix this: `)
|
||||
+ Green(`npm run setup`)
|
||||
Red("API_URL env variable is missing, cobalt api can't start.")
|
||||
)
|
||||
}
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
import { getVersion } from "@imput/version-info";
|
||||
import { services } from "./processing/service-config.js";
|
||||
import { supportsReusePort } from "./misc/cluster.js";
|
||||
|
||||
const version = await getVersion();
|
||||
|
||||
|
@ -13,6 +14,7 @@ const enabledServices = new Set(Object.keys(services).filter(e => {
|
|||
const env = {
|
||||
apiURL: process.env.API_URL || '',
|
||||
apiPort: process.env.API_PORT || 9000,
|
||||
tunnelPort: process.env.API_PORT || 9000,
|
||||
|
||||
listenAddress: process.env.API_LISTEN_ADDRESS,
|
||||
freebindCIDR: process.platform === 'linux' && process.env.FREEBIND_CIDR,
|
||||
|
@ -45,7 +47,8 @@ const env = {
|
|||
|
||||
apiKeyURL: process.env.API_KEY_URL && new URL(process.env.API_KEY_URL),
|
||||
authRequired: process.env.API_AUTH_REQUIRED === '1',
|
||||
|
||||
redisURL: process.env.API_REDIS_URL,
|
||||
instanceCount: (process.env.API_INSTANCE_COUNT && parseInt(process.env.API_INSTANCE_COUNT)) || 1,
|
||||
keyReloadInterval: 900,
|
||||
|
||||
enabledServices,
|
||||
|
@ -54,6 +57,23 @@ const env = {
|
|||
const genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36";
|
||||
const cobaltUserAgent = `cobalt/${version} (+https://github.com/imputnet/cobalt)`;
|
||||
|
||||
export const setTunnelPort = (port) => env.tunnelPort = port;
|
||||
export const isCluster = env.instanceCount > 1;
|
||||
|
||||
if (env.sessionEnabled && env.jwtSecret.length < 16) {
|
||||
throw new Error("JWT_SECRET env is too short (must be at least 16 characters long)");
|
||||
}
|
||||
|
||||
if (env.instanceCount > 1 && !env.redisURL) {
|
||||
throw new Error("API_REDIS_URL is required when API_INSTANCE_COUNT is >= 2");
|
||||
} else if (env.instanceCount > 1 && !await supportsReusePort()) {
|
||||
console.error('API_INSTANCE_COUNT is not supported in your environment. to use this env, your node.js');
|
||||
console.error('version must be >= 23.1.0, and you must be running a recent enough version of linux');
|
||||
console.error('(or other OS that supports it). for more info, see `reusePort` option on');
|
||||
console.error('https://nodejs.org/api/net.html#serverlistenoptions-callback');
|
||||
throw new Error('SO_REUSEPORT is not supported');
|
||||
}
|
||||
|
||||
export {
|
||||
env,
|
||||
genericUserAgent,
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
import cors from "cors";
|
||||
import http from "node:http";
|
||||
import rateLimit from "express-rate-limit";
|
||||
import { setGlobalDispatcher, ProxyAgent } from "undici";
|
||||
import { getCommit, getBranch, getRemote, getVersion } from "@imput/version-info";
|
||||
|
@ -7,17 +8,18 @@ import jwt from "../security/jwt.js";
|
|||
import stream from "../stream/stream.js";
|
||||
import match from "../processing/match.js";
|
||||
|
||||
import { env } from "../config.js";
|
||||
import { env, isCluster, setTunnelPort } from "../config.js";
|
||||
import { extract } from "../processing/url.js";
|
||||
import { languageCode } from "../misc/utils.js";
|
||||
import { Bright, Cyan } from "../misc/console-text.js";
|
||||
import { generateHmac, generateSalt } from "../misc/crypto.js";
|
||||
import { Green, Bright, Cyan } from "../misc/console-text.js";
|
||||
import { hashHmac } from "../security/secrets.js";
|
||||
import { createStore } from "../store/redis-ratelimit.js";
|
||||
import { randomizeCiphers } from "../misc/randomize-ciphers.js";
|
||||
import { verifyTurnstileToken } from "../security/turnstile.js";
|
||||
import { friendlyServiceName } from "../processing/service-alias.js";
|
||||
import { verifyStream, getInternalStream } from "../stream/manage.js";
|
||||
import { createResponse, normalizeRequest, getIP } from "../processing/request.js";
|
||||
import * as APIKeys from "../security/api-keys.js";
|
||||
import * as Cookies from "../processing/cookie/manager.js";
|
||||
|
||||
const git = {
|
||||
branch: await getBranch(),
|
||||
|
@ -29,7 +31,6 @@ const version = await getVersion();
|
|||
|
||||
const acceptRegex = /^application\/json(; charset=utf-8)?$/;
|
||||
|
||||
const ipSalt = generateSalt();
|
||||
const corsConfig = env.corsWildcard ? {} : {
|
||||
origin: env.corsURL,
|
||||
optionsSuccessStatus: 200
|
||||
|
@ -40,7 +41,7 @@ const fail = (res, code, context) => {
|
|||
res.status(status).json(body);
|
||||
}
|
||||
|
||||
export const runAPI = (express, app, __dirname) => {
|
||||
export const runAPI = async (express, app, __dirname, isPrimary = true) => {
|
||||
const startTime = new Date();
|
||||
const startTimestamp = startTime.getTime();
|
||||
|
||||
|
@ -68,31 +69,36 @@ export const runAPI = (express, app, __dirname) => {
|
|||
return res.status(status).json(body);
|
||||
};
|
||||
|
||||
const keyGenerator = (req) => hashHmac(getIP(req), 'rate').toString('base64url');
|
||||
|
||||
const sessionLimiter = rateLimit({
|
||||
windowMs: 60000,
|
||||
max: 10,
|
||||
standardHeaders: true,
|
||||
limit: 10,
|
||||
standardHeaders: 'draft-6',
|
||||
legacyHeaders: false,
|
||||
keyGenerator: req => generateHmac(getIP(req), ipSalt),
|
||||
keyGenerator,
|
||||
store: await createStore('session'),
|
||||
handler: handleRateExceeded
|
||||
});
|
||||
|
||||
const apiLimiter = rateLimit({
|
||||
windowMs: env.rateLimitWindow * 1000,
|
||||
max: (req) => req.rateLimitMax || env.rateLimitMax,
|
||||
standardHeaders: true,
|
||||
limit: (req) => req.rateLimitMax || env.rateLimitMax,
|
||||
standardHeaders: 'draft-6',
|
||||
legacyHeaders: false,
|
||||
keyGenerator: req => req.rateLimitKey || generateHmac(getIP(req), ipSalt),
|
||||
keyGenerator: req => req.rateLimitKey || keyGenerator(req),
|
||||
store: await createStore('api'),
|
||||
handler: handleRateExceeded
|
||||
})
|
||||
|
||||
const apiTunnelLimiter = rateLimit({
|
||||
windowMs: env.rateLimitWindow * 1000,
|
||||
max: (req) => req.rateLimitMax || env.rateLimitMax,
|
||||
standardHeaders: true,
|
||||
limit: (req) => req.rateLimitMax || env.rateLimitMax,
|
||||
standardHeaders: 'draft-6',
|
||||
legacyHeaders: false,
|
||||
keyGenerator: req => req.rateLimitKey || generateHmac(getIP(req), ipSalt),
|
||||
handler: (req, res) => {
|
||||
keyGenerator: req => req.rateLimitKey || keyGenerator(req),
|
||||
store: await createStore('tunnel'),
|
||||
handler: (_, res) => {
|
||||
return res.sendStatus(429)
|
||||
}
|
||||
})
|
||||
|
@ -158,19 +164,20 @@ export const runAPI = (express, app, __dirname) => {
|
|||
return fail(res, "error.api.auth.jwt.missing");
|
||||
}
|
||||
|
||||
if (!authorization.startsWith("Bearer ") || authorization.length > 256) {
|
||||
if (authorization.length >= 256) {
|
||||
return fail(res, "error.api.auth.jwt.invalid");
|
||||
}
|
||||
|
||||
const verifyJwt = jwt.verify(
|
||||
authorization.split("Bearer ", 2)[1]
|
||||
);
|
||||
|
||||
if (!verifyJwt) {
|
||||
const [ type, token, ...rest ] = authorization.split(" ");
|
||||
if (!token || type.toLowerCase() !== 'bearer' || rest.length) {
|
||||
return fail(res, "error.api.auth.jwt.invalid");
|
||||
}
|
||||
|
||||
req.rateLimitKey = generateHmac(req.header("Authorization"), ipSalt);
|
||||
if (!jwt.verify(token)) {
|
||||
return fail(res, "error.api.auth.jwt.invalid");
|
||||
}
|
||||
|
||||
req.rateLimitKey = hashHmac(token, 'rate');
|
||||
} catch {
|
||||
return fail(res, "error.api.generic");
|
||||
}
|
||||
|
@ -220,16 +227,11 @@ export const runAPI = (express, app, __dirname) => {
|
|||
|
||||
app.post('/', async (req, res) => {
|
||||
const request = req.body;
|
||||
const lang = languageCode(req);
|
||||
|
||||
if (!request.url) {
|
||||
return fail(res, "error.api.link.missing");
|
||||
}
|
||||
|
||||
if (request.youtubeDubBrowserLang) {
|
||||
request.youtubeDubLang = lang;
|
||||
}
|
||||
|
||||
const { success, data: normalizedRequest } = await normalizeRequest(request);
|
||||
if (!success) {
|
||||
return fail(res, "error.api.invalid_body");
|
||||
|
@ -261,7 +263,7 @@ export const runAPI = (express, app, __dirname) => {
|
|||
}
|
||||
})
|
||||
|
||||
app.get('/tunnel', apiTunnelLimiter, (req, res) => {
|
||||
app.get('/tunnel', apiTunnelLimiter, async (req, res) => {
|
||||
const id = String(req.query.id);
|
||||
const exp = String(req.query.exp);
|
||||
const sig = String(req.query.sig);
|
||||
|
@ -280,7 +282,7 @@ export const runAPI = (express, app, __dirname) => {
|
|||
return res.status(200).end();
|
||||
}
|
||||
|
||||
const streamInfo = verifyStream(id, sig, exp, sec, iv);
|
||||
const streamInfo = await verifyStream(id, sig, exp, sec, iv);
|
||||
if (!streamInfo?.service) {
|
||||
return res.status(streamInfo.status).end();
|
||||
}
|
||||
|
@ -292,7 +294,7 @@ export const runAPI = (express, app, __dirname) => {
|
|||
return stream(res, streamInfo);
|
||||
})
|
||||
|
||||
app.get('/itunnel', (req, res) => {
|
||||
const itunnelHandler = (req, res) => {
|
||||
if (!req.ip.endsWith('127.0.0.1')) {
|
||||
return res.sendStatus(403);
|
||||
}
|
||||
|
@ -311,8 +313,10 @@ export const runAPI = (express, app, __dirname) => {
|
|||
...Object.entries(req.headers)
|
||||
]);
|
||||
|
||||
return stream(res, { type: 'internal', ...streamInfo });
|
||||
})
|
||||
return stream(res, { type: 'internal', data: streamInfo });
|
||||
};
|
||||
|
||||
app.get('/itunnel', itunnelHandler);
|
||||
|
||||
app.get('/', (_, res) => {
|
||||
res.type('json');
|
||||
|
@ -343,24 +347,48 @@ export const runAPI = (express, app, __dirname) => {
|
|||
setGlobalDispatcher(new ProxyAgent(env.externalProxy))
|
||||
}
|
||||
|
||||
if (env.apiKeyURL) {
|
||||
APIKeys.setup(env.apiKeyURL);
|
||||
http.createServer(app).listen({
|
||||
port: env.apiPort,
|
||||
host: env.listenAddress,
|
||||
reusePort: env.instanceCount > 1 || undefined
|
||||
}, () => {
|
||||
if (isPrimary) {
|
||||
console.log(`\n` +
|
||||
Bright(Cyan("cobalt ")) + Bright("API ^ω^") + "\n" +
|
||||
|
||||
"~~~~~~\n" +
|
||||
Bright("version: ") + version + "\n" +
|
||||
Bright("commit: ") + git.commit + "\n" +
|
||||
Bright("branch: ") + git.branch + "\n" +
|
||||
Bright("remote: ") + git.remote + "\n" +
|
||||
Bright("start time: ") + startTime.toUTCString() + "\n" +
|
||||
"~~~~~~\n" +
|
||||
|
||||
Bright("url: ") + Bright(Cyan(env.apiURL)) + "\n" +
|
||||
Bright("port: ") + env.apiPort + "\n"
|
||||
);
|
||||
}
|
||||
|
||||
if (env.apiKeyURL) {
|
||||
APIKeys.setup(env.apiKeyURL);
|
||||
}
|
||||
|
||||
if (env.cookiePath) {
|
||||
Cookies.setup(env.cookiePath);
|
||||
}
|
||||
});
|
||||
|
||||
if (isCluster) {
|
||||
const istreamer = express();
|
||||
istreamer.get('/itunnel', itunnelHandler);
|
||||
const server = istreamer.listen({
|
||||
port: 0,
|
||||
host: '127.0.0.1',
|
||||
exclusive: true
|
||||
}, () => {
|
||||
const { port } = server.address();
|
||||
console.log(`${Green('[✓]')} cobalt sub-instance running on 127.0.0.1:${port}`);
|
||||
setTunnelPort(port);
|
||||
});
|
||||
}
|
||||
|
||||
app.listen(env.apiPort, env.listenAddress, () => {
|
||||
console.log(`\n` +
|
||||
Bright(Cyan("cobalt ")) + Bright("API ^ω^") + "\n" +
|
||||
|
||||
"~~~~~~\n" +
|
||||
Bright("version: ") + version + "\n" +
|
||||
Bright("commit: ") + git.commit + "\n" +
|
||||
Bright("branch: ") + git.branch + "\n" +
|
||||
Bright("remote: ") + git.remote + "\n" +
|
||||
Bright("start time: ") + startTime.toUTCString() + "\n" +
|
||||
"~~~~~~\n" +
|
||||
|
||||
Bright("url: ") + Bright(Cyan(env.apiURL)) + "\n" +
|
||||
Bright("port: ") + env.apiPort + "\n"
|
||||
)
|
||||
})
|
||||
}
|
||||
|
|
71
api/src/misc/cluster.js
Normal file
71
api/src/misc/cluster.js
Normal file
|
@ -0,0 +1,71 @@
|
|||
import cluster from "node:cluster";
|
||||
import net from "node:net";
|
||||
import { syncSecrets } from "../security/secrets.js";
|
||||
import { env, isCluster } from "../config.js";
|
||||
|
||||
export { isPrimary, isWorker } from "node:cluster";
|
||||
|
||||
export const supportsReusePort = async () => {
|
||||
try {
|
||||
await new Promise((resolve, reject) => {
|
||||
const server = net.createServer().listen({ port: 0, reusePort: true });
|
||||
server.on('listening', () => server.close(resolve));
|
||||
server.on('error', (err) => (server.close(), reject(err)));
|
||||
});
|
||||
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export const initCluster = async () => {
|
||||
if (cluster.isPrimary) {
|
||||
for (let i = 1; i < env.instanceCount; ++i) {
|
||||
cluster.fork();
|
||||
}
|
||||
}
|
||||
|
||||
await syncSecrets();
|
||||
}
|
||||
|
||||
export const broadcast = (message) => {
|
||||
if (!isCluster || !cluster.isPrimary || !cluster.workers) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const worker of Object.values(cluster.workers)) {
|
||||
worker.send(message);
|
||||
}
|
||||
}
|
||||
|
||||
export const send = (message) => {
|
||||
if (!isCluster) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (cluster.isPrimary) {
|
||||
return broadcast(message);
|
||||
} else {
|
||||
return process.send(message);
|
||||
}
|
||||
}
|
||||
|
||||
export const waitFor = (key) => {
|
||||
return new Promise(resolve => {
|
||||
const listener = (message) => {
|
||||
if (key in message) {
|
||||
process.off('message', listener);
|
||||
return resolve(message);
|
||||
}
|
||||
}
|
||||
|
||||
process.on('message', listener);
|
||||
});
|
||||
}
|
||||
|
||||
export const mainOnMessage = (cb) => {
|
||||
for (const worker of Object.values(cluster.workers)) {
|
||||
worker.on('message', cb);
|
||||
}
|
||||
}
|
|
@ -1,23 +1,36 @@
|
|||
function t(color, tt) {
|
||||
return color + tt + "\x1b[0m"
|
||||
const ANSI = {
|
||||
RESET: "\x1b[0m",
|
||||
BRIGHT: "\x1b[1m",
|
||||
RED: "\x1b[31m",
|
||||
GREEN: "\x1b[32m",
|
||||
CYAN: "\x1b[36m",
|
||||
YELLOW: "\x1b[93m"
|
||||
}
|
||||
|
||||
export function Bright(tt) {
|
||||
return t("\x1b[1m", tt)
|
||||
function wrap(color, text) {
|
||||
if (!ANSI[color.toUpperCase()]) {
|
||||
throw "invalid color";
|
||||
}
|
||||
|
||||
return ANSI[color.toUpperCase()] + text + ANSI.RESET;
|
||||
}
|
||||
|
||||
export function Red(tt) {
|
||||
return t("\x1b[31m", tt)
|
||||
export function Bright(text) {
|
||||
return wrap('bright', text);
|
||||
}
|
||||
|
||||
export function Green(tt) {
|
||||
return t("\x1b[32m", tt)
|
||||
export function Red(text) {
|
||||
return wrap('red', text);
|
||||
}
|
||||
|
||||
export function Cyan(tt) {
|
||||
return t("\x1b[36m", tt)
|
||||
export function Green(text) {
|
||||
return wrap('green', text);
|
||||
}
|
||||
|
||||
export function Yellow(tt) {
|
||||
return t("\x1b[93m", tt)
|
||||
export function Cyan(text) {
|
||||
return wrap('cyan', text);
|
||||
}
|
||||
|
||||
export function Yellow(text) {
|
||||
return wrap('yellow', text);
|
||||
}
|
||||
|
|
|
@ -1,15 +1,7 @@
|
|||
import { createHmac, createCipheriv, createDecipheriv, randomBytes } from "crypto";
|
||||
import { createCipheriv, createDecipheriv } from "crypto";
|
||||
|
||||
const algorithm = "aes256";
|
||||
|
||||
export function generateSalt() {
|
||||
return randomBytes(64).toString('hex');
|
||||
}
|
||||
|
||||
export function generateHmac(str, salt) {
|
||||
return createHmac("sha256", salt).update(str).digest("base64url");
|
||||
}
|
||||
|
||||
export function encryptStream(plaintext, iv, secret) {
|
||||
const buff = Buffer.from(JSON.stringify(plaintext));
|
||||
const key = Buffer.from(secret, "base64url");
|
||||
|
|
|
@ -41,4 +41,4 @@ export async function runTest(url, params, expect) {
|
|||
if (result.body.status === 'tunnel') {
|
||||
// TODO: stream testing
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,55 +1,16 @@
|
|||
const forbiddenCharsString = ['}', '{', '%', '>', '<', '^', ';', '`', '$', '"', "@", '='];
|
||||
const redirectStatuses = new Set([301, 302, 303, 307, 308]);
|
||||
|
||||
export function metadataManager(obj) {
|
||||
const keys = Object.keys(obj);
|
||||
const tags = [
|
||||
"album",
|
||||
"copyright",
|
||||
"title",
|
||||
"artist",
|
||||
"track",
|
||||
"date"
|
||||
]
|
||||
let commands = []
|
||||
|
||||
for (const i in keys) {
|
||||
if (tags.includes(keys[i]))
|
||||
commands.push('-metadata', `${keys[i]}=${obj[keys[i]]}`)
|
||||
}
|
||||
return commands;
|
||||
}
|
||||
|
||||
export function cleanString(string) {
|
||||
for (const i in forbiddenCharsString) {
|
||||
string = string.replaceAll("/", "_")
|
||||
.replaceAll(forbiddenCharsString[i], '')
|
||||
}
|
||||
return string;
|
||||
}
|
||||
export function verifyLanguageCode(code) {
|
||||
const langCode = String(code.slice(0, 2).toLowerCase());
|
||||
if (RegExp(/[a-z]{2}/).test(code)) {
|
||||
return langCode
|
||||
}
|
||||
return "en"
|
||||
}
|
||||
export function languageCode(req) {
|
||||
if (req.header('Accept-Language')) {
|
||||
return verifyLanguageCode(req.header('Accept-Language'))
|
||||
}
|
||||
return "en"
|
||||
}
|
||||
export function cleanHTML(html) {
|
||||
let clean = html.replace(/ {4}/g, '');
|
||||
clean = clean.replace(/\n/g, '');
|
||||
return clean
|
||||
}
|
||||
|
||||
export function getRedirectingURL(url) {
|
||||
return fetch(url, { redirect: 'manual' }).then((r) => {
|
||||
if ([301, 302, 303].includes(r.status) && r.headers.has('location'))
|
||||
export async function getRedirectingURL(url, dispatcher) {
|
||||
const location = await fetch(url, {
|
||||
redirect: 'manual',
|
||||
dispatcher,
|
||||
}).then((r) => {
|
||||
if (redirectStatuses.has(r.status) && r.headers.has('location')) {
|
||||
return r.headers.get('location');
|
||||
}
|
||||
}).catch(() => null);
|
||||
|
||||
return location;
|
||||
}
|
||||
|
||||
export function merge(a, b) {
|
||||
|
@ -76,3 +37,7 @@ export function splitFilenameExtension(filename) {
|
|||
return [ parts.join('.'), ext ]
|
||||
}
|
||||
}
|
||||
|
||||
export function zip(a, b) {
|
||||
return a.map((value, i) => [ value, b[i] ]);
|
||||
}
|
||||
|
|
|
@ -4,16 +4,24 @@ export default class Cookie {
|
|||
constructor(input) {
|
||||
assert(typeof input === 'object');
|
||||
this._values = {};
|
||||
this.set(input)
|
||||
|
||||
for (const [ k, v ] of Object.entries(input))
|
||||
this.set(k, v);
|
||||
}
|
||||
set(values) {
|
||||
Object.entries(values).forEach(
|
||||
([ key, value ]) => this._values[key] = value
|
||||
)
|
||||
|
||||
set(key, value) {
|
||||
const old = this._values[key];
|
||||
if (old === value)
|
||||
return false;
|
||||
|
||||
this._values[key] = value;
|
||||
return true;
|
||||
}
|
||||
|
||||
unset(keys) {
|
||||
for (const key of keys) delete this._values[key]
|
||||
}
|
||||
|
||||
static fromString(str) {
|
||||
const obj = {};
|
||||
|
||||
|
@ -25,12 +33,15 @@ export default class Cookie {
|
|||
|
||||
return new Cookie(obj)
|
||||
}
|
||||
|
||||
toString() {
|
||||
return Object.entries(this._values).map(([ name, value ]) => `${name}=${value}`).join('; ')
|
||||
}
|
||||
|
||||
toJSON() {
|
||||
return this.toString()
|
||||
}
|
||||
|
||||
values() {
|
||||
return Object.freeze({ ...this._values })
|
||||
}
|
||||
|
|
|
@ -1,50 +1,145 @@
|
|||
import Cookie from './cookie.js';
|
||||
|
||||
import { readFile, writeFile } from 'fs/promises';
|
||||
import { Red, Green, Yellow } from '../../misc/console-text.js';
|
||||
import { parse as parseSetCookie, splitCookiesString } from 'set-cookie-parser';
|
||||
import { env } from '../../config.js';
|
||||
import * as cluster from '../../misc/cluster.js';
|
||||
import { isCluster } from '../../config.js';
|
||||
|
||||
const WRITE_INTERVAL = 60000,
|
||||
cookiePath = env.cookiePath,
|
||||
COUNTER = Symbol('counter');
|
||||
const WRITE_INTERVAL = 60000;
|
||||
const VALID_SERVICES = new Set([
|
||||
'instagram',
|
||||
'instagram_bearer',
|
||||
'reddit',
|
||||
'twitter',
|
||||
'youtube',
|
||||
'youtube_oauth'
|
||||
]);
|
||||
|
||||
const invalidCookies = {};
|
||||
let cookies = {}, dirty = false, intervalId;
|
||||
|
||||
const setup = async () => {
|
||||
try {
|
||||
if (!cookiePath) return;
|
||||
|
||||
cookies = await readFile(cookiePath, 'utf8');
|
||||
cookies = JSON.parse(cookies);
|
||||
intervalId = setInterval(writeChanges, WRITE_INTERVAL)
|
||||
} catch { /* no cookies for you */ }
|
||||
}
|
||||
|
||||
setup();
|
||||
|
||||
function writeChanges() {
|
||||
function writeChanges(cookiePath) {
|
||||
if (!dirty) return;
|
||||
dirty = false;
|
||||
|
||||
writeFile(cookiePath, JSON.stringify(cookies, null, 4)).catch(() => {
|
||||
clearInterval(intervalId)
|
||||
const cookieData = JSON.stringify({ ...cookies, ...invalidCookies }, null, 4);
|
||||
writeFile(cookiePath, cookieData).catch((e) => {
|
||||
console.warn(`${Yellow('[!]')} failed writing updated cookies to storage`);
|
||||
console.warn(e);
|
||||
clearInterval(intervalId);
|
||||
intervalId = null;
|
||||
})
|
||||
}
|
||||
|
||||
export function getCookie(service) {
|
||||
if (!cookies[service] || !cookies[service].length) return;
|
||||
const setupMain = async (cookiePath) => {
|
||||
try {
|
||||
cookies = await readFile(cookiePath, 'utf8');
|
||||
cookies = JSON.parse(cookies);
|
||||
for (const serviceName in cookies) {
|
||||
if (!VALID_SERVICES.has(serviceName)) {
|
||||
console.warn(`${Yellow('[!]')} ignoring unknown service in cookie file: ${serviceName}`);
|
||||
} else if (!Array.isArray(cookies[serviceName])) {
|
||||
console.warn(`${Yellow('[!]')} ${serviceName} in cookies file is not an array, ignoring it`);
|
||||
} else if (cookies[serviceName].some(c => typeof c !== 'string')) {
|
||||
console.warn(`${Yellow('[!]')} some cookie for ${serviceName} contains non-string value in cookies file`);
|
||||
} else continue;
|
||||
|
||||
let n;
|
||||
if (cookies[service][COUNTER] === undefined) {
|
||||
n = cookies[service][COUNTER] = 0
|
||||
} else {
|
||||
++cookies[service][COUNTER]
|
||||
n = (cookies[service][COUNTER] %= cookies[service].length)
|
||||
invalidCookies[serviceName] = cookies[serviceName];
|
||||
delete cookies[serviceName];
|
||||
}
|
||||
|
||||
if (!intervalId) {
|
||||
intervalId = setInterval(() => writeChanges(cookiePath), WRITE_INTERVAL);
|
||||
}
|
||||
|
||||
cluster.broadcast({ cookies });
|
||||
|
||||
console.log(`${Green('[✓]')} cookies loaded successfully!`);
|
||||
} catch (e) {
|
||||
console.error(`${Yellow('[!]')} failed to load cookies.`);
|
||||
console.error('error:', e);
|
||||
}
|
||||
}
|
||||
|
||||
const setupWorker = async () => {
|
||||
cookies = (await cluster.waitFor('cookies')).cookies;
|
||||
}
|
||||
|
||||
export const loadFromFile = async (path) => {
|
||||
if (cluster.isPrimary) {
|
||||
await setupMain(path);
|
||||
} else if (cluster.isWorker) {
|
||||
await setupWorker();
|
||||
}
|
||||
|
||||
const cookie = cookies[service][n];
|
||||
if (typeof cookie === 'string') cookies[service][n] = Cookie.fromString(cookie);
|
||||
dirty = false;
|
||||
}
|
||||
|
||||
return cookies[service][n]
|
||||
export const setup = async (path) => {
|
||||
await loadFromFile(path);
|
||||
|
||||
if (isCluster) {
|
||||
const messageHandler = (message) => {
|
||||
if ('cookieUpdate' in message) {
|
||||
const { cookieUpdate } = message;
|
||||
|
||||
if (cluster.isPrimary) {
|
||||
dirty = true;
|
||||
cluster.broadcast({ cookieUpdate });
|
||||
}
|
||||
|
||||
const { service, idx, cookie } = cookieUpdate;
|
||||
cookies[service][idx] = cookie;
|
||||
}
|
||||
}
|
||||
|
||||
if (cluster.isPrimary) {
|
||||
cluster.mainOnMessage(messageHandler);
|
||||
} else {
|
||||
process.on('message', messageHandler);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function getCookie(service) {
|
||||
if (!VALID_SERVICES.has(service)) {
|
||||
console.error(
|
||||
`${Red('[!]')} ${service} not in allowed services list for cookies.`
|
||||
+ ' if adding a new cookie type, include it there.'
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!cookies[service] || !cookies[service].length) return;
|
||||
|
||||
const idx = Math.floor(Math.random() * cookies[service].length);
|
||||
|
||||
const cookie = cookies[service][idx];
|
||||
if (typeof cookie === 'string') {
|
||||
cookies[service][idx] = Cookie.fromString(cookie);
|
||||
}
|
||||
|
||||
cookies[service][idx].meta = { service, idx };
|
||||
return cookies[service][idx];
|
||||
}
|
||||
|
||||
export function updateCookieValues(cookie, values) {
|
||||
let changed = false;
|
||||
|
||||
for (const [ key, value ] of Object.entries(values)) {
|
||||
changed = cookie.set(key, value) || changed;
|
||||
}
|
||||
|
||||
if (changed && cookie.meta) {
|
||||
dirty = true;
|
||||
if (isCluster) {
|
||||
const message = { cookieUpdate: { ...cookie.meta, cookie } };
|
||||
cluster.send(message);
|
||||
}
|
||||
}
|
||||
|
||||
return changed;
|
||||
}
|
||||
|
||||
export function updateCookie(cookie, headers) {
|
||||
|
@ -57,10 +152,6 @@ export function updateCookie(cookie, headers) {
|
|||
|
||||
cookie.unset(parsed.filter(c => c.expires < new Date()).map(c => c.name));
|
||||
parsed.filter(c => !c.expires || c.expires > new Date()).forEach(c => values[c.name] = c.value);
|
||||
|
||||
updateCookieValues(cookie, values);
|
||||
}
|
||||
|
||||
export function updateCookieValues(cookie, values) {
|
||||
cookie.set(values);
|
||||
if (Object.keys(values).length) dirty = true
|
||||
}
|
||||
|
|
|
@ -1,3 +1,13 @@
|
|||
const illegalCharacters = ['}', '{', '%', '>', '<', '^', ';', ':', '`', '$', '"', "@", '=', '?', '|', '*'];
|
||||
|
||||
const sanitizeString = (string) => {
|
||||
for (const i in illegalCharacters) {
|
||||
string = string.replaceAll("/", "_").replaceAll("\\", "_")
|
||||
.replaceAll(illegalCharacters[i], '')
|
||||
}
|
||||
return string;
|
||||
}
|
||||
|
||||
export default (f, style, isAudioOnly, isAudioMuted) => {
|
||||
let filename = '';
|
||||
|
||||
|
@ -5,7 +15,11 @@ export default (f, style, isAudioOnly, isAudioMuted) => {
|
|||
let classicTags = [...infoBase];
|
||||
let basicTags = [];
|
||||
|
||||
const title = `${f.title} - ${f.author}`;
|
||||
let title = sanitizeString(f.title);
|
||||
|
||||
if (f.author) {
|
||||
title += ` - ${sanitizeString(f.author)}`;
|
||||
}
|
||||
|
||||
if (f.resolution) {
|
||||
classicTags.push(f.resolution);
|
||||
|
|
|
@ -9,13 +9,14 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
|
|||
let action,
|
||||
responseType = "tunnel",
|
||||
defaultParams = {
|
||||
u: r.urls,
|
||||
url: r.urls,
|
||||
headers: r.headers,
|
||||
service: host,
|
||||
filename: r.filenameAttributes ?
|
||||
createFilename(r.filenameAttributes, filenameStyle, isAudioOnly, isAudioMuted) : r.filename,
|
||||
fileMetadata: !disableMetadata ? r.fileMetadata : false,
|
||||
requestIP
|
||||
requestIP,
|
||||
originalRequest: r.originalRequest
|
||||
},
|
||||
params = {};
|
||||
|
||||
|
@ -24,7 +25,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
|
|||
else if (r.isGif && twitterGif) action = "gif";
|
||||
else if (isAudioOnly) action = "audio";
|
||||
else if (isAudioMuted) action = "muteVideo";
|
||||
else if (r.isM3U8) action = "m3u8";
|
||||
else if (r.isHLS) action = "hls";
|
||||
else action = "video";
|
||||
|
||||
if (action === "picker" || action === "audio") {
|
||||
|
@ -47,27 +48,29 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
|
|||
});
|
||||
|
||||
case "photo":
|
||||
responseType = "redirect";
|
||||
params = { type: "proxy" };
|
||||
break;
|
||||
|
||||
case "gif":
|
||||
params = { type: "gif" };
|
||||
break;
|
||||
|
||||
case "m3u8":
|
||||
case "hls":
|
||||
params = {
|
||||
type: Array.isArray(r.urls) ? "merge" : "remux"
|
||||
type: Array.isArray(r.urls) ? "merge" : "remux",
|
||||
isHLS: true,
|
||||
}
|
||||
break;
|
||||
|
||||
case "muteVideo":
|
||||
let muteType = "mute";
|
||||
if (Array.isArray(r.urls) && !r.isM3U8) {
|
||||
if (Array.isArray(r.urls) && !r.isHLS) {
|
||||
muteType = "proxy";
|
||||
}
|
||||
params = {
|
||||
type: muteType,
|
||||
u: Array.isArray(r.urls) ? r.urls[0] : r.urls
|
||||
url: Array.isArray(r.urls) ? r.urls[0] : r.urls,
|
||||
isHLS: r.isHLS
|
||||
}
|
||||
if (host === "reddit" && r.typeId === "redirect") {
|
||||
responseType = "redirect";
|
||||
|
@ -82,6 +85,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
|
|||
case "threads":
|
||||
case "snapchat":
|
||||
case "bsky":
|
||||
case "xiaohongshu":
|
||||
params = { picker: r.picker };
|
||||
break;
|
||||
|
||||
|
@ -93,14 +97,15 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
|
|||
}
|
||||
params = {
|
||||
picker: r.picker,
|
||||
u: createStream({
|
||||
url: createStream({
|
||||
service: "tiktok",
|
||||
type: audioStreamType,
|
||||
u: r.urls,
|
||||
url: r.urls,
|
||||
headers: r.headers,
|
||||
filename: r.audioFilename,
|
||||
filename: `${r.audioFilename}.${audioFormat}`,
|
||||
isAudioOnly: true,
|
||||
audioFormat,
|
||||
audioBitrate
|
||||
})
|
||||
}
|
||||
break;
|
||||
|
@ -141,11 +146,11 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
|
|||
case "ok":
|
||||
case "vk":
|
||||
case "tiktok":
|
||||
case "xiaohongshu":
|
||||
params = { type: "proxy" };
|
||||
break;
|
||||
|
||||
case "facebook":
|
||||
case "vine":
|
||||
case "instagram":
|
||||
case "tumblr":
|
||||
case "pinterest":
|
||||
|
@ -162,7 +167,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
|
|||
case "audio":
|
||||
if (audioIgnore.includes(host) || (host === "reddit" && r.typeId === "redirect")) {
|
||||
return createResponse("error", {
|
||||
code: "error.api.fetch.empty"
|
||||
code: "error.api.service.audio_not_supported"
|
||||
})
|
||||
}
|
||||
|
||||
|
@ -186,18 +191,20 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
|
|||
}
|
||||
}
|
||||
|
||||
if (r.isM3U8 || host === "vimeo") {
|
||||
if (r.isHLS || host === "vimeo") {
|
||||
copy = false;
|
||||
processType = "audio";
|
||||
}
|
||||
|
||||
params = {
|
||||
type: processType,
|
||||
u: Array.isArray(r.urls) ? r.urls[1] : r.urls,
|
||||
url: Array.isArray(r.urls) ? r.urls[1] : r.urls,
|
||||
|
||||
audioBitrate,
|
||||
audioCopy: copy,
|
||||
audioFormat,
|
||||
|
||||
isHLS: r.isHLS,
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
|
|
@ -19,7 +19,6 @@ import tumblr from "./services/tumblr.js";
|
|||
import vimeo from "./services/vimeo.js";
|
||||
import soundcloud from "./services/soundcloud.js";
|
||||
import instagram from "./services/instagram.js";
|
||||
import vine from "./services/vine.js";
|
||||
import pinterest from "./services/pinterest.js";
|
||||
import streamable from "./services/streamable.js";
|
||||
import twitch from "./services/twitch.js";
|
||||
|
@ -30,6 +29,7 @@ import loom from "./services/loom.js";
|
|||
import threads from "./services/threads.js";
|
||||
import facebook from "./services/facebook.js";
|
||||
import bluesky from "./services/bluesky.js";
|
||||
import xiaohongshu from "./services/xiaohongshu.js";
|
||||
|
||||
let freebind;
|
||||
|
||||
|
@ -79,8 +79,9 @@ export default async function({ host, patternMatch, params }) {
|
|||
|
||||
case "vk":
|
||||
r = await vk({
|
||||
userId: patternMatch.userId,
|
||||
ownerId: patternMatch.ownerId,
|
||||
videoId: patternMatch.videoId,
|
||||
accessKey: patternMatch.accessKey,
|
||||
quality: params.videoQuality
|
||||
});
|
||||
break;
|
||||
|
@ -98,13 +99,14 @@ export default async function({ host, patternMatch, params }) {
|
|||
|
||||
case "youtube":
|
||||
let fetchInfo = {
|
||||
dispatcher,
|
||||
id: patternMatch.id.slice(0, 11),
|
||||
quality: params.videoQuality,
|
||||
format: params.youtubeVideoCodec,
|
||||
isAudioOnly,
|
||||
isAudioMuted,
|
||||
dubLang: params.youtubeDubLang,
|
||||
dispatcher
|
||||
youtubeHLS: params.youtubeHLS,
|
||||
}
|
||||
|
||||
if (url.hostname === "music.youtube.com" || isAudioOnly) {
|
||||
|
@ -128,7 +130,7 @@ export default async function({ host, patternMatch, params }) {
|
|||
case "tiktok":
|
||||
r = await tiktok({
|
||||
postId: patternMatch.postId,
|
||||
id: patternMatch.id,
|
||||
shortLink: patternMatch.shortLink,
|
||||
fullAudio: params.tiktokFullAudio,
|
||||
isAudioOnly,
|
||||
h265: params.tiktokH265,
|
||||
|
@ -175,12 +177,6 @@ export default async function({ host, patternMatch, params }) {
|
|||
})
|
||||
break;
|
||||
|
||||
case "vine":
|
||||
r = await vine({
|
||||
id: patternMatch.id
|
||||
});
|
||||
break;
|
||||
|
||||
case "pinterest":
|
||||
r = await pinterest({
|
||||
id: patternMatch.id,
|
||||
|
@ -249,7 +245,17 @@ export default async function({ host, patternMatch, params }) {
|
|||
case "bsky":
|
||||
r = await bluesky({
|
||||
...patternMatch,
|
||||
alwaysProxy: params.alwaysProxy
|
||||
alwaysProxy: params.alwaysProxy,
|
||||
dispatcher
|
||||
});
|
||||
break;
|
||||
|
||||
case "xiaohongshu":
|
||||
r = await xiaohongshu({
|
||||
...patternMatch,
|
||||
h265: params.tiktokH265,
|
||||
isAudioOnly,
|
||||
dispatcher,
|
||||
});
|
||||
break;
|
||||
|
||||
|
|
|
@ -37,7 +37,7 @@ export function createResponse(responseType, responseData) {
|
|||
|
||||
case "redirect":
|
||||
response = {
|
||||
url: responseData?.u,
|
||||
url: responseData?.url,
|
||||
filename: responseData?.filename
|
||||
}
|
||||
break;
|
||||
|
@ -52,7 +52,7 @@ export function createResponse(responseType, responseData) {
|
|||
case "picker":
|
||||
response = {
|
||||
picker: responseData?.picker,
|
||||
audio: responseData?.u,
|
||||
audio: responseData?.url,
|
||||
audioFilename: responseData?.filename
|
||||
}
|
||||
break;
|
||||
|
|
|
@ -1,7 +1,5 @@
|
|||
import { z } from "zod";
|
||||
|
||||
import { normalizeURL } from "./url.js";
|
||||
import { verifyLanguageCode } from "../misc/utils.js";
|
||||
|
||||
export const apiSchema = z.object({
|
||||
url: z.string()
|
||||
|
@ -33,15 +31,21 @@ export const apiSchema = z.object({
|
|||
).default("1080"),
|
||||
|
||||
youtubeDubLang: z.string()
|
||||
.length(2)
|
||||
.transform(verifyLanguageCode)
|
||||
.min(2)
|
||||
.max(8)
|
||||
.regex(/^[0-9a-zA-Z\-]+$/)
|
||||
.optional(),
|
||||
|
||||
// TODO: remove this variable as it's no longer used
|
||||
// and is kept for schema compatibility reasons
|
||||
youtubeDubBrowserLang: z.boolean().default(false),
|
||||
|
||||
alwaysProxy: z.boolean().default(false),
|
||||
disableMetadata: z.boolean().default(false),
|
||||
tiktokFullAudio: z.boolean().default(false),
|
||||
tiktokH265: z.boolean().default(false),
|
||||
twitterGif: z.boolean().default(true),
|
||||
youtubeDubBrowserLang: z.boolean().default(false),
|
||||
|
||||
youtubeHLS: z.boolean().default(false),
|
||||
})
|
||||
.strict();
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
import UrlPattern from "url-pattern";
|
||||
|
||||
export const audioIgnore = ["vk", "ok", "loom"];
|
||||
export const hlsExceptions = ["dailymotion", "vimeo", "rutube", "bsky"];
|
||||
export const hlsExceptions = ["dailymotion", "vimeo", "rutube", "bsky", "youtube"];
|
||||
|
||||
export const services = {
|
||||
bilibili: {
|
||||
|
@ -30,7 +30,7 @@ export const services = {
|
|||
"reel/:id",
|
||||
"share/:shareType/:id"
|
||||
],
|
||||
subdomains: ["web"],
|
||||
subdomains: ["web", "m"],
|
||||
altDomains: ["fb.watch"],
|
||||
},
|
||||
instagram: {
|
||||
|
@ -46,7 +46,7 @@ export const services = {
|
|||
altDomains: ["ddinstagram.com"],
|
||||
},
|
||||
loom: {
|
||||
patterns: ["share/:id"],
|
||||
patterns: ["share/:id", "embed/:id"],
|
||||
},
|
||||
ok: {
|
||||
patterns: [
|
||||
|
@ -115,10 +115,10 @@ export const services = {
|
|||
tiktok: {
|
||||
patterns: [
|
||||
":user/video/:postId",
|
||||
":id",
|
||||
"t/:id",
|
||||
":shortLink",
|
||||
"t/:shortLink",
|
||||
":user/photo/:postId",
|
||||
"v/:id.html"
|
||||
"v/:postId.html"
|
||||
],
|
||||
subdomains: ["vt", "vm", "m"],
|
||||
},
|
||||
|
@ -147,10 +147,6 @@ export const services = {
|
|||
subdomains: ["mobile"],
|
||||
altDomains: ["x.com", "vxtwitter.com", "fixvx.com"],
|
||||
},
|
||||
vine: {
|
||||
patterns: ["v/:id"],
|
||||
tld: "co",
|
||||
},
|
||||
vimeo: {
|
||||
patterns: [
|
||||
":id",
|
||||
|
@ -162,11 +158,25 @@ export const services = {
|
|||
},
|
||||
vk: {
|
||||
patterns: [
|
||||
"video:userId_:videoId",
|
||||
"clip:userId_:videoId",
|
||||
"clips:duplicate?z=clip:userId_:videoId"
|
||||
"video:ownerId_:videoId",
|
||||
"clip:ownerId_:videoId",
|
||||
"clips:duplicate?z=clip:ownerId_:videoId",
|
||||
"videos:duplicate?z=video:ownerId_:videoId",
|
||||
"video:ownerId_:videoId_:accessKey",
|
||||
"clip:ownerId_:videoId_:accessKey",
|
||||
"clips:duplicate?z=clip:ownerId_:videoId_:accessKey",
|
||||
"videos:duplicate?z=video:ownerId_:videoId_:accessKey"
|
||||
],
|
||||
subdomains: ["m"],
|
||||
altDomains: ["vkvideo.ru", "vk.ru"],
|
||||
},
|
||||
xiaohongshu: {
|
||||
patterns: [
|
||||
"explore/:id?xsec_token=:token",
|
||||
"discovery/item/:id?xsec_token=:token",
|
||||
"a/:shareId"
|
||||
],
|
||||
altDomains: ["xhslink.com"],
|
||||
},
|
||||
youtube: {
|
||||
patterns: [
|
||||
|
|
|
@ -36,13 +36,13 @@ export const testers = {
|
|||
|| pattern.shortLink?.length <= 16,
|
||||
|
||||
"streamable": pattern =>
|
||||
pattern.id?.length === 6,
|
||||
pattern.id?.length <= 6,
|
||||
|
||||
"threads": pattern =>
|
||||
pattern.user?.length <= 33 && pattern.id?.length <= 32,
|
||||
|
||||
"tiktok": pattern =>
|
||||
pattern.postId?.length <= 21 || pattern.id?.length <= 13,
|
||||
pattern.postId?.length <= 21 || pattern.shortLink?.length <= 13,
|
||||
|
||||
"tumblr": pattern =>
|
||||
pattern.id?.length < 21
|
||||
|
@ -58,11 +58,9 @@ export const testers = {
|
|||
pattern.id?.length <= 11
|
||||
&& (!pattern.password || pattern.password.length < 16),
|
||||
|
||||
"vine": pattern =>
|
||||
pattern.id?.length <= 12,
|
||||
|
||||
"vk": pattern =>
|
||||
pattern.userId?.length <= 10 && pattern.videoId?.length <= 10,
|
||||
(pattern.ownerId?.length <= 10 && pattern.videoId?.length <= 10) ||
|
||||
(pattern.ownerId?.length <= 10 && pattern.videoId?.length <= 10 && pattern.videoId?.accessKey <= 18),
|
||||
|
||||
"youtube": pattern =>
|
||||
pattern.id?.length <= 11,
|
||||
|
@ -76,4 +74,8 @@ export const testers = {
|
|||
|
||||
"bsky": pattern =>
|
||||
pattern.user?.length <= 128 && pattern.post?.length <= 128,
|
||||
|
||||
"xiaohongshu": pattern =>
|
||||
pattern.id?.length <= 24 && pattern.token?.length <= 64
|
||||
|| pattern.shareId?.length <= 12,
|
||||
}
|
||||
|
|
|
@ -2,12 +2,19 @@ import HLS from "hls-parser";
|
|||
import { cobaltUserAgent } from "../../config.js";
|
||||
import { createStream } from "../../stream/manage.js";
|
||||
|
||||
const extractVideo = async ({ media, filename }) => {
|
||||
const urlMasterHLS = media?.playlist;
|
||||
if (!urlMasterHLS) return { error: "fetch.empty" };
|
||||
if (!urlMasterHLS.startsWith("https://video.bsky.app/")) return { error: "fetch.empty" };
|
||||
const extractVideo = async ({ media, filename, dispatcher }) => {
|
||||
let urlMasterHLS = media?.playlist;
|
||||
|
||||
const masterHLS = await fetch(urlMasterHLS)
|
||||
if (!urlMasterHLS || !urlMasterHLS.startsWith("https://video.bsky.app/")) {
|
||||
return { error: "fetch.empty" };
|
||||
}
|
||||
|
||||
urlMasterHLS = urlMasterHLS.replace(
|
||||
"video.bsky.app/watch/",
|
||||
"video.cdn.bsky.app/hls/"
|
||||
);
|
||||
|
||||
const masterHLS = await fetch(urlMasterHLS, { dispatcher })
|
||||
.then(r => {
|
||||
if (r.status !== 200) return;
|
||||
return r.text();
|
||||
|
@ -26,7 +33,7 @@ const extractVideo = async ({ media, filename }) => {
|
|||
urls: videoURL,
|
||||
filename: `${filename}.mp4`,
|
||||
audioFilename: `${filename}_audio`,
|
||||
isM3U8: true,
|
||||
isHLS: true,
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -48,7 +55,7 @@ const extractImages = ({ getPost, filename, alwaysProxy }) => {
|
|||
let proxiedImage = createStream({
|
||||
service: "bluesky",
|
||||
type: "proxy",
|
||||
u: url,
|
||||
url,
|
||||
filename: `${filename}_${i + 1}.jpg`,
|
||||
});
|
||||
|
||||
|
@ -64,7 +71,25 @@ const extractImages = ({ getPost, filename, alwaysProxy }) => {
|
|||
return { picker };
|
||||
}
|
||||
|
||||
export default async function ({ user, post, alwaysProxy }) {
|
||||
const extractGif = ({ url, filename }) => {
|
||||
const gifUrl = new URL(url);
|
||||
|
||||
if (!gifUrl || gifUrl.hostname !== "media.tenor.com") {
|
||||
return { error: "fetch.empty" };
|
||||
}
|
||||
|
||||
// remove downscaling params from gif url
|
||||
// such as "?hh=498&ww=498"
|
||||
gifUrl.search = "";
|
||||
|
||||
return {
|
||||
urls: gifUrl,
|
||||
isPhoto: true,
|
||||
filename: `${filename}.gif`,
|
||||
}
|
||||
}
|
||||
|
||||
export default async function ({ user, post, alwaysProxy, dispatcher }) {
|
||||
const apiEndpoint = new URL("https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread?depth=0&parentHeight=0");
|
||||
apiEndpoint.searchParams.set(
|
||||
"uri",
|
||||
|
@ -73,8 +98,9 @@ export default async function ({ user, post, alwaysProxy }) {
|
|||
|
||||
const getPost = await fetch(apiEndpoint, {
|
||||
headers: {
|
||||
"user-agent": cobaltUserAgent
|
||||
}
|
||||
"user-agent": cobaltUserAgent,
|
||||
},
|
||||
dispatcher
|
||||
}).then(r => r.json()).catch(() => {});
|
||||
|
||||
if (!getPost) return { error: "fetch.empty" };
|
||||
|
@ -87,29 +113,44 @@ export default async function ({ user, post, alwaysProxy }) {
|
|||
case "InvalidRequest":
|
||||
return { error: "link.unsupported" };
|
||||
default:
|
||||
return { error: "fetch.empty" };
|
||||
return { error: "content.post.unavailable" };
|
||||
}
|
||||
}
|
||||
|
||||
const embedType = getPost?.thread?.post?.embed?.$type;
|
||||
const filename = `bluesky_${user}_${post}`;
|
||||
|
||||
if (embedType === "app.bsky.embed.video#view") {
|
||||
return extractVideo({
|
||||
media: getPost.thread?.post?.embed,
|
||||
filename,
|
||||
})
|
||||
}
|
||||
switch (embedType) {
|
||||
case "app.bsky.embed.video#view":
|
||||
return extractVideo({
|
||||
media: getPost.thread?.post?.embed,
|
||||
filename,
|
||||
});
|
||||
|
||||
if (embedType === "app.bsky.embed.recordWithMedia#view") {
|
||||
return extractVideo({
|
||||
media: getPost.thread?.post?.embed?.media,
|
||||
filename,
|
||||
})
|
||||
}
|
||||
case "app.bsky.embed.images#view":
|
||||
return extractImages({
|
||||
getPost,
|
||||
filename,
|
||||
alwaysProxy
|
||||
});
|
||||
|
||||
if (embedType === "app.bsky.embed.images#view") {
|
||||
return extractImages({ getPost, filename, alwaysProxy });
|
||||
case "app.bsky.embed.external#view":
|
||||
return extractGif({
|
||||
url: getPost?.thread?.post?.embed?.external?.uri,
|
||||
filename,
|
||||
});
|
||||
|
||||
case "app.bsky.embed.recordWithMedia#view":
|
||||
if (getPost?.thread?.post?.embed?.media?.$type === "app.bsky.embed.external#view") {
|
||||
return extractGif({
|
||||
url: getPost?.thread?.post?.embed?.media?.external?.uri,
|
||||
filename,
|
||||
});
|
||||
}
|
||||
return extractVideo({
|
||||
media: getPost.thread?.post?.embed?.media,
|
||||
filename,
|
||||
});
|
||||
}
|
||||
|
||||
return { error: "fetch.empty" };
|
||||
|
|
|
@ -92,7 +92,7 @@ export default async function({ id }) {
|
|||
|
||||
return {
|
||||
urls: bestQuality.uri,
|
||||
isM3U8: true,
|
||||
isHLS: true,
|
||||
filenameAttributes: {
|
||||
service: 'dailymotion',
|
||||
id: media.xid,
|
||||
|
|
|
@ -177,7 +177,7 @@ export default function(obj) {
|
|||
if (alwaysProxy) proxyFile = createStream({
|
||||
service: "instagram",
|
||||
type: "proxy",
|
||||
u: url,
|
||||
url,
|
||||
filename: `instagram_${id}_${i + 1}.${itemExt}`
|
||||
});
|
||||
|
||||
|
@ -189,7 +189,7 @@ export default function(obj) {
|
|||
thumb: createStream({
|
||||
service: "instagram",
|
||||
type: "proxy",
|
||||
u: e.node?.display_url,
|
||||
url: e.node?.display_url,
|
||||
filename: `instagram_${id}_${i + 1}.jpg`
|
||||
})
|
||||
}
|
||||
|
@ -230,7 +230,7 @@ export default function(obj) {
|
|||
if (alwaysProxy) proxyFile = createStream({
|
||||
service: "instagram",
|
||||
type: "proxy",
|
||||
u: url,
|
||||
url,
|
||||
filename: `instagram_${id}_${i + 1}.${itemExt}`
|
||||
});
|
||||
|
||||
|
@ -242,7 +242,7 @@ export default function(obj) {
|
|||
thumb: createStream({
|
||||
service: "instagram",
|
||||
type: "proxy",
|
||||
u: imageUrl,
|
||||
url: imageUrl,
|
||||
filename: `instagram_${id}_${i + 1}.jpg`
|
||||
})
|
||||
}
|
||||
|
@ -266,6 +266,7 @@ export default function(obj) {
|
|||
}
|
||||
|
||||
async function getPost(id, alwaysProxy) {
|
||||
const hasData = (data) => data && data.gql_data !== null;
|
||||
let data, result;
|
||||
try {
|
||||
const cookie = getCookie('instagram');
|
||||
|
@ -282,16 +283,16 @@ export default function(obj) {
|
|||
if (media_id && token) data = await requestMobileApi(media_id, { token });
|
||||
|
||||
// mobile api (no cookie, cookie)
|
||||
if (media_id && !data) data = await requestMobileApi(media_id);
|
||||
if (media_id && cookie && !data) data = await requestMobileApi(media_id, { cookie });
|
||||
if (media_id && !hasData(data)) data = await requestMobileApi(media_id);
|
||||
if (media_id && cookie && !hasData(data)) data = await requestMobileApi(media_id, { cookie });
|
||||
|
||||
// html embed (no cookie, cookie)
|
||||
if (!data) data = await requestHTML(id);
|
||||
if (!data && cookie) data = await requestHTML(id, cookie);
|
||||
if (!hasData(data)) data = await requestHTML(id);
|
||||
if (!hasData(data) && cookie) data = await requestHTML(id, cookie);
|
||||
|
||||
// web app graphql api (no cookie, cookie)
|
||||
if (!data) data = await requestGQL(id);
|
||||
if (!data && cookie) data = await requestGQL(id, cookie);
|
||||
if (!hasData(data)) data = await requestGQL(id);
|
||||
if (!hasData(data) && cookie) data = await requestGQL(id, cookie);
|
||||
} catch {}
|
||||
|
||||
if (!data) return { error: "fetch.fail" };
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
import { genericUserAgent, env } from "../../config.js";
|
||||
import { cleanString } from "../../misc/utils.js";
|
||||
|
||||
const resolutions = {
|
||||
"ultra": "2160",
|
||||
|
@ -44,8 +43,8 @@ export default async function(o) {
|
|||
let bestVideo = videos.find(v => resolutions[v.name] === quality) || videos[videos.length - 1];
|
||||
|
||||
let fileMetadata = {
|
||||
title: cleanString(videoData.movie.title.trim()),
|
||||
author: cleanString((videoData.author?.name || videoData.compilationTitle).trim()),
|
||||
title: videoData.movie.title.trim(),
|
||||
author: (videoData.author?.name || videoData.compilationTitle).trim(),
|
||||
}
|
||||
|
||||
if (bestVideo) return {
|
||||
|
|
|
@ -1,7 +1,5 @@
|
|||
import HLS from "hls-parser";
|
||||
|
||||
import { env } from "../../config.js";
|
||||
import { cleanString } from "../../misc/utils.js";
|
||||
|
||||
async function requestJSON(url) {
|
||||
try {
|
||||
|
@ -35,6 +33,10 @@ export default async function(obj) {
|
|||
const play = await requestJSON(requestURL);
|
||||
if (!play) return { error: "fetch.fail" };
|
||||
|
||||
if (play.detail?.type === "blocking_rule") {
|
||||
return { error: "content.video.region" };
|
||||
}
|
||||
|
||||
if (play.detail || !play.video_balancer) return { error: "fetch.empty" };
|
||||
if (play.live_streams?.hls) return { error: "content.video.live" };
|
||||
|
||||
|
@ -59,13 +61,13 @@ export default async function(obj) {
|
|||
});
|
||||
|
||||
const fileMetadata = {
|
||||
title: cleanString(play.title.trim()),
|
||||
artist: cleanString(play.author.name.trim()),
|
||||
title: play.title.trim(),
|
||||
artist: play.author.name.trim(),
|
||||
}
|
||||
|
||||
return {
|
||||
urls: matchingQuality.uri,
|
||||
isM3U8: true,
|
||||
isHLS: true,
|
||||
filenameAttributes: {
|
||||
service: "rutube",
|
||||
id: obj.id,
|
||||
|
|
|
@ -73,7 +73,7 @@ async function getStory(username, storyId, alwaysProxy) {
|
|||
const proxy = createStream({
|
||||
service: "snapchat",
|
||||
type: "proxy",
|
||||
u: snapUrl,
|
||||
url: snapUrl,
|
||||
filename: `snapchat_${username}_${snap.timestampInSec.value}.${snapExt}`,
|
||||
});
|
||||
|
||||
|
@ -81,7 +81,7 @@ async function getStory(username, storyId, alwaysProxy) {
|
|||
if (snapType === "video") thumbProxy = createStream({
|
||||
service: "snapchat",
|
||||
type: "proxy",
|
||||
u: snap.snapUrls.mediaPreviewUrl.value,
|
||||
url: snap.snapUrls.mediaPreviewUrl.value,
|
||||
});
|
||||
|
||||
if (alwaysProxy) snapUrl = proxy;
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
import { env } from "../../config.js";
|
||||
import { cleanString } from "../../misc/utils.js";
|
||||
|
||||
const cachedID = {
|
||||
version: '',
|
||||
|
@ -63,7 +62,17 @@ export default async function(obj) {
|
|||
|
||||
if (!json) return { error: "fetch.fail" };
|
||||
|
||||
if (!json.media.transcodings) return { error: "fetch.empty" };
|
||||
if (json?.policy === "BLOCK") {
|
||||
return { error: "content.region" };
|
||||
}
|
||||
|
||||
if (json?.policy === "SNIP") {
|
||||
return { error: "content.paid" };
|
||||
}
|
||||
|
||||
if (!json?.media?.transcodings || !json?.media?.transcodings.length === 0) {
|
||||
return { error: "fetch.empty" };
|
||||
}
|
||||
|
||||
let bestAudio = "opus",
|
||||
selectedStream = json.media.transcodings.find(v => v.preset === "opus_0_0"),
|
||||
|
@ -75,6 +84,10 @@ export default async function(obj) {
|
|||
bestAudio = "mp3"
|
||||
}
|
||||
|
||||
if (!selectedStream) {
|
||||
return { error: "fetch.empty" };
|
||||
}
|
||||
|
||||
let fileUrlBase = selectedStream.url;
|
||||
let fileUrl = `${fileUrlBase}${fileUrlBase.includes("?") ? "&" : "?"}client_id=${clientId}&track_authorization=${json.track_authorization}`;
|
||||
|
||||
|
@ -91,8 +104,8 @@ export default async function(obj) {
|
|||
if (!file) return { error: "fetch.empty" };
|
||||
|
||||
let fileMetadata = {
|
||||
title: cleanString(json.title.trim()),
|
||||
artist: cleanString(json.user.username.trim()),
|
||||
title: json.title.trim(),
|
||||
artist: json.user.username.trim(),
|
||||
}
|
||||
|
||||
return {
|
||||
|
|
|
@ -12,7 +12,7 @@ export default async function(obj) {
|
|||
let postId = obj.postId;
|
||||
|
||||
if (!postId) {
|
||||
let html = await fetch(`${shortDomain}${obj.id}`, {
|
||||
let html = await fetch(`${shortDomain}${obj.shortLink}`, {
|
||||
redirect: "manual",
|
||||
headers: {
|
||||
"user-agent": genericUserAgent.split(' Chrome/1')[0]
|
||||
|
@ -24,13 +24,13 @@ export default async function(obj) {
|
|||
if (html.startsWith('<a href="https://')) {
|
||||
const extractedURL = html.split('<a href="')[1].split('?')[0];
|
||||
const { patternMatch } = extract(extractedURL);
|
||||
postId = patternMatch.postId
|
||||
postId = patternMatch.postId;
|
||||
}
|
||||
}
|
||||
if (!postId) return { error: "fetch.short_link" };
|
||||
|
||||
// should always be /video/, even for photos
|
||||
const res = await fetch(`https://tiktok.com/@i/video/${postId}`, {
|
||||
const res = await fetch(`https://www.tiktok.com/@i/video/${postId}`, {
|
||||
headers: {
|
||||
"user-agent": genericUserAgent,
|
||||
cookie,
|
||||
|
@ -44,20 +44,39 @@ export default async function(obj) {
|
|||
try {
|
||||
const json = html
|
||||
.split('<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">')[1]
|
||||
.split('</script>')[0]
|
||||
const data = JSON.parse(json)
|
||||
detail = data["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"]
|
||||
.split('</script>')[0];
|
||||
|
||||
const data = JSON.parse(json);
|
||||
const videoDetail = data["__DEFAULT_SCOPE__"]["webapp.video-detail"];
|
||||
|
||||
if (!videoDetail) throw "no video detail found";
|
||||
|
||||
// status_deleted or etc
|
||||
if (videoDetail.statusMsg) {
|
||||
return { error: "content.post.unavailable"};
|
||||
}
|
||||
|
||||
detail = videoDetail?.itemInfo?.itemStruct;
|
||||
} catch {
|
||||
return { error: "fetch.fail" };
|
||||
}
|
||||
|
||||
if (detail.isContentClassified) {
|
||||
return { error: "content.post.age" };
|
||||
}
|
||||
|
||||
if (!detail.author) {
|
||||
return { error: "fetch.empty" };
|
||||
}
|
||||
|
||||
let video, videoFilename, audioFilename, audio, images,
|
||||
filenameBase = `tiktok_${detail.author.uniqueId}_${postId}`,
|
||||
filenameBase = `tiktok_${detail.author?.uniqueId}_${postId}`,
|
||||
bestAudio; // will get defaulted to m4a later on in match-action
|
||||
|
||||
images = detail.imagePost?.images;
|
||||
|
||||
let playAddr = detail.video.playAddr;
|
||||
let playAddr = detail.video?.playAddr;
|
||||
|
||||
if (obj.h265) {
|
||||
const h265PlayAddr = detail?.video?.bitrateInfo?.find(b => b.CodecType.includes("h265"))?.PlayAddr.UrlList[0]
|
||||
playAddr = h265PlayAddr || playAddr
|
||||
|
@ -102,7 +121,7 @@ export default async function(obj) {
|
|||
if (obj.alwaysProxy) url = createStream({
|
||||
service: "tiktok",
|
||||
type: "proxy",
|
||||
u: url,
|
||||
url,
|
||||
filename: `${filenameBase}_photo_${i + 1}.jpg`
|
||||
})
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
import psl from "psl";
|
||||
import psl from "@imput/psl";
|
||||
|
||||
const API_KEY = 'jrsCWX1XDuVxAFO4GkK147syAoN8BJZ5voz8tS80bPcj26Vc5Z';
|
||||
const API_BASE = 'https://api-http2.tumblr.com';
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
import { env } from "../../config.js";
|
||||
import { cleanString } from '../../misc/utils.js';
|
||||
|
||||
const gqlURL = "https://gql.twitch.tv/gql";
|
||||
const clientIdHead = { "client-id": "kimne78kx3ncx6brgo4mv6wki5h1ko" };
|
||||
|
@ -73,13 +72,13 @@ export default async function (obj) {
|
|||
token: req_token[0].data.clip.playbackAccessToken.value
|
||||
})}`,
|
||||
fileMetadata: {
|
||||
title: cleanString(clipMetadata.title.trim()),
|
||||
title: clipMetadata.title.trim(),
|
||||
artist: `Twitch Clip by @${clipMetadata.broadcaster.login}, clipped by @${clipMetadata.curator.login}`,
|
||||
},
|
||||
filenameAttributes: {
|
||||
service: "twitch",
|
||||
id: clipMetadata.id,
|
||||
title: cleanString(clipMetadata.title.trim()),
|
||||
title: clipMetadata.title.trim(),
|
||||
author: `${clipMetadata.broadcaster.login}, clipped by ${clipMetadata.curator.login}`,
|
||||
qualityLabel: `${format.quality}p`,
|
||||
extension: 'mp4'
|
||||
|
|
|
@ -159,10 +159,10 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
|
|||
|
||||
const getFileExt = (url) => new URL(url).pathname.split(".", 2)[1];
|
||||
|
||||
const proxyMedia = (u, filename) => createStream({
|
||||
const proxyMedia = (url, filename) => createStream({
|
||||
service: "twitter",
|
||||
type: "proxy",
|
||||
u, filename,
|
||||
url, filename,
|
||||
})
|
||||
|
||||
switch (media?.length) {
|
||||
|
@ -208,7 +208,7 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
|
|||
|
||||
let url = bestQuality(content.video_info.variants);
|
||||
const shouldRenderGif = content.type === "animated_gif" && toGif;
|
||||
const videoFilename = `twitter_${id}_${i + 1}.mp4`;
|
||||
const videoFilename = `twitter_${id}_${i + 1}.${shouldRenderGif ? "gif" : "mp4"}`;
|
||||
|
||||
let type = "video";
|
||||
if (shouldRenderGif) type = "gif";
|
||||
|
@ -217,7 +217,7 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
|
|||
url = createStream({
|
||||
service: "twitter",
|
||||
type: shouldRenderGif ? "gif" : "remux",
|
||||
u: url,
|
||||
url,
|
||||
filename: videoFilename,
|
||||
})
|
||||
} else if (alwaysProxy) {
|
||||
|
|
|
@ -1,7 +1,6 @@
|
|||
import HLS from "hls-parser";
|
||||
|
||||
import { env } from "../../config.js";
|
||||
import { cleanString, merge } from '../../misc/utils.js';
|
||||
import { merge } from '../../misc/utils.js';
|
||||
|
||||
const resolutionMatch = {
|
||||
"3840": 2160,
|
||||
|
@ -122,7 +121,7 @@ const getHLS = async (configURL, obj) => {
|
|||
|
||||
return {
|
||||
urls,
|
||||
isM3U8: true,
|
||||
isHLS: true,
|
||||
filenameAttributes: {
|
||||
resolution: `${bestQuality.resolution.width}x${bestQuality.resolution.height}`,
|
||||
qualityLabel: `${resolutionMatch[bestQuality.resolution.width]}p`,
|
||||
|
@ -152,8 +151,8 @@ export default async function(obj) {
|
|||
}
|
||||
|
||||
const fileMetadata = {
|
||||
title: cleanString(info.name),
|
||||
artist: cleanString(info.user.name),
|
||||
title: info.name,
|
||||
artist: info.user.name,
|
||||
};
|
||||
|
||||
return merge(
|
||||
|
|
|
@ -1,15 +0,0 @@
|
|||
export default async function(obj) {
|
||||
let post = await fetch(`https://archive.vine.co/posts/${obj.id}.json`)
|
||||
.then(r => r.json())
|
||||
.catch(() => {});
|
||||
|
||||
if (!post) return { error: "fetch.empty" };
|
||||
|
||||
if (post.videoUrl) return {
|
||||
urls: post.videoUrl.replace("http://", "https://"),
|
||||
filename: `vine_${obj.id}.mp4`,
|
||||
audioFilename: `vine_${obj.id}_audio`
|
||||
}
|
||||
|
||||
return { error: "fetch.empty" }
|
||||
}
|
|
@ -1,63 +1,140 @@
|
|||
import { cleanString } from "../../misc/utils.js";
|
||||
import { genericUserAgent, env } from "../../config.js";
|
||||
import { env } from "../../config.js";
|
||||
|
||||
const resolutions = ["2160", "1440", "1080", "720", "480", "360", "240"];
|
||||
const resolutions = ["2160", "1440", "1080", "720", "480", "360", "240", "144"];
|
||||
|
||||
export default async function(o) {
|
||||
let html, url, quality = o.quality === "max" ? 2160 : o.quality;
|
||||
const oauthUrl = "https://oauth.vk.com/oauth/get_anonym_token";
|
||||
const apiUrl = "https://api.vk.com/method";
|
||||
|
||||
html = await fetch(`https://vk.com/video${o.userId}_${o.videoId}`, {
|
||||
headers: {
|
||||
"user-agent": genericUserAgent
|
||||
}
|
||||
})
|
||||
.then(r => r.arrayBuffer())
|
||||
.catch(() => {});
|
||||
const clientId = "51552953";
|
||||
const clientSecret = "qgr0yWwXCrsxA1jnRtRX";
|
||||
|
||||
if (!html) return { error: "fetch.fail" };
|
||||
// used in stream/shared.js for accessing media files
|
||||
export const vkClientAgent = "com.vk.vkvideo.prod/822 (iPhone, iOS 16.7.7, iPhone10,4, Scale/2.0) SAK/1.119";
|
||||
|
||||
// decode cyrillic from windows-1251 because vk still uses apis from prehistoric times
|
||||
let decoder = new TextDecoder('windows-1251');
|
||||
html = decoder.decode(html);
|
||||
const cachedToken = {
|
||||
token: "",
|
||||
expiry: 0,
|
||||
device_id: "",
|
||||
};
|
||||
|
||||
if (!html.includes(`{"lang":`)) return { error: "fetch.empty" };
|
||||
|
||||
let js = JSON.parse('{"lang":' + html.split(`{"lang":`)[1].split(']);')[0]);
|
||||
|
||||
if (Number(js.mvData.is_active_live) !== 0) {
|
||||
return { error: "content.video.live" };
|
||||
const getToken = async () => {
|
||||
if (cachedToken.expiry - 10 > Math.floor(new Date().getTime() / 1000)) {
|
||||
return cachedToken.token;
|
||||
}
|
||||
|
||||
if (js.mvData.duration > env.durationLimit) {
|
||||
const randomDeviceId = crypto.randomUUID().toUpperCase();
|
||||
|
||||
const anonymOauth = new URL(oauthUrl);
|
||||
anonymOauth.searchParams.set("client_id", clientId);
|
||||
anonymOauth.searchParams.set("client_secret", clientSecret);
|
||||
anonymOauth.searchParams.set("device_id", randomDeviceId);
|
||||
|
||||
const oauthResponse = await fetch(anonymOauth.toString(), {
|
||||
headers: {
|
||||
"user-agent": vkClientAgent,
|
||||
}
|
||||
}).then(r => {
|
||||
if (r.status === 200) {
|
||||
return r.json();
|
||||
}
|
||||
});
|
||||
|
||||
if (!oauthResponse) return;
|
||||
|
||||
if (oauthResponse?.token && oauthResponse?.expired_at && typeof oauthResponse?.expired_at === "number") {
|
||||
cachedToken.token = oauthResponse.token;
|
||||
cachedToken.expiry = oauthResponse.expired_at;
|
||||
cachedToken.device_id = randomDeviceId;
|
||||
}
|
||||
|
||||
if (!cachedToken.token) return;
|
||||
|
||||
return cachedToken.token;
|
||||
}
|
||||
|
||||
const getVideo = async (ownerId, videoId, accessKey) => {
|
||||
const video = await fetch(`${apiUrl}/video.get`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/x-www-form-urlencoded; charset=utf-8",
|
||||
"user-agent": vkClientAgent,
|
||||
},
|
||||
body: new URLSearchParams({
|
||||
anonymous_token: cachedToken.token,
|
||||
device_id: cachedToken.device_id,
|
||||
lang: "en",
|
||||
v: "5.244",
|
||||
videos: `${ownerId}_${videoId}${accessKey ? `_${accessKey}` : ''}`
|
||||
}).toString()
|
||||
})
|
||||
.then(r => {
|
||||
if (r.status === 200) {
|
||||
return r.json();
|
||||
}
|
||||
});
|
||||
|
||||
return video;
|
||||
}
|
||||
|
||||
export default async function ({ ownerId, videoId, accessKey, quality }) {
|
||||
const token = await getToken();
|
||||
if (!token) return { error: "fetch.fail" };
|
||||
|
||||
const videoGet = await getVideo(ownerId, videoId, accessKey);
|
||||
|
||||
if (!videoGet || !videoGet.response || videoGet.response.items.length !== 1) {
|
||||
return { error: "fetch.empty" };
|
||||
}
|
||||
|
||||
const video = videoGet.response.items[0];
|
||||
|
||||
if (video.restriction) {
|
||||
const title = video.restriction.title;
|
||||
if (title.endsWith("country") || title.endsWith("region.")) {
|
||||
return { error: "content.video.region" };
|
||||
}
|
||||
if (title === "Processing video") {
|
||||
return { error: "fetch.empty" };
|
||||
}
|
||||
return { error: "content.video.unavailable" };
|
||||
}
|
||||
|
||||
if (!video.files || !video.duration) {
|
||||
return { error: "fetch.fail" };
|
||||
}
|
||||
|
||||
if (video.duration > env.durationLimit) {
|
||||
return { error: "content.too_long" };
|
||||
}
|
||||
|
||||
for (let i in resolutions) {
|
||||
if (js.player.params[0][`url${resolutions[i]}`]) {
|
||||
quality = resolutions[i];
|
||||
const userQuality = quality === "max" ? resolutions[0] : quality;
|
||||
let pickedQuality;
|
||||
|
||||
for (const resolution of resolutions) {
|
||||
if (video.files[`mp4_${resolution}`] && +resolution <= +userQuality) {
|
||||
pickedQuality = resolution;
|
||||
break
|
||||
}
|
||||
}
|
||||
if (Number(quality) > Number(o.quality)) quality = o.quality;
|
||||
|
||||
url = js.player.params[0][`url${quality}`];
|
||||
const url = video.files[`mp4_${pickedQuality}`];
|
||||
|
||||
let fileMetadata = {
|
||||
title: cleanString(js.player.params[0].md_title.trim()),
|
||||
author: cleanString(js.player.params[0].md_author.trim()),
|
||||
if (!url) return { error: "fetch.fail" };
|
||||
|
||||
const fileMetadata = {
|
||||
title: video.title.trim(),
|
||||
}
|
||||
|
||||
if (url) return {
|
||||
return {
|
||||
urls: url,
|
||||
fileMetadata,
|
||||
filenameAttributes: {
|
||||
service: "vk",
|
||||
id: `${o.userId}_${o.videoId}`,
|
||||
id: `${ownerId}_${videoId}${accessKey ? `_${accessKey}` : ''}`,
|
||||
title: fileMetadata.title,
|
||||
author: fileMetadata.author,
|
||||
resolution: `${quality}p`,
|
||||
qualityLabel: `${quality}p`,
|
||||
resolution: `${pickedQuality}p`,
|
||||
qualityLabel: `${pickedQuality}p`,
|
||||
extension: "mp4"
|
||||
}
|
||||
}
|
||||
return { error: "fetch.empty" }
|
||||
}
|
||||
|
|
116
api/src/processing/services/xiaohongshu.js
Normal file
116
api/src/processing/services/xiaohongshu.js
Normal file
|
@ -0,0 +1,116 @@
|
|||
import { extract, normalizeURL } from "../url.js";
|
||||
import { genericUserAgent } from "../../config.js";
|
||||
import { createStream } from "../../stream/manage.js";
|
||||
import { getRedirectingURL } from "../../misc/utils.js";
|
||||
|
||||
const https = (url) => {
|
||||
return url.replace(/^http:/i, 'https:');
|
||||
}
|
||||
|
||||
export default async function ({ id, token, shareId, h265, isAudioOnly, dispatcher }) {
|
||||
let noteId = id;
|
||||
let xsecToken = token;
|
||||
|
||||
if (!noteId) {
|
||||
const extractedURL = await getRedirectingURL(
|
||||
`https://xhslink.com/a/${shareId}`,
|
||||
dispatcher
|
||||
);
|
||||
|
||||
if (extractedURL) {
|
||||
const { patternMatch } = extract(normalizeURL(extractedURL));
|
||||
|
||||
if (patternMatch) {
|
||||
noteId = patternMatch.id;
|
||||
xsecToken = patternMatch.token;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!noteId || !xsecToken) return { error: "fetch.short_link" };
|
||||
|
||||
const res = await fetch(`https://www.xiaohongshu.com/explore/${noteId}?xsec_token=${xsecToken}`, {
|
||||
headers: {
|
||||
"user-agent": genericUserAgent,
|
||||
},
|
||||
dispatcher,
|
||||
});
|
||||
|
||||
const html = await res.text();
|
||||
|
||||
let note;
|
||||
try {
|
||||
const initialState = html
|
||||
.split('<script>window.__INITIAL_STATE__=')[1]
|
||||
.split('</script>')[0]
|
||||
.replace(/:\s*undefined/g, ":null");
|
||||
|
||||
const data = JSON.parse(initialState);
|
||||
|
||||
const noteInfo = data?.note?.noteDetailMap;
|
||||
if (!noteInfo) throw "no note detail map";
|
||||
|
||||
const currentNote = noteInfo[noteId];
|
||||
if (!currentNote) throw "no current note in detail map";
|
||||
|
||||
note = currentNote.note;
|
||||
} catch {}
|
||||
|
||||
if (!note) return { error: "fetch.empty" };
|
||||
|
||||
const video = note.video;
|
||||
const images = note.imageList;
|
||||
|
||||
const filenameBase = `xiaohongshu_${noteId}`;
|
||||
|
||||
if (video) {
|
||||
const videoFilename = `${filenameBase}.mp4`;
|
||||
const audioFilename = `${filenameBase}_audio`;
|
||||
|
||||
let videoURL;
|
||||
|
||||
if (h265 && !isAudioOnly && video.consumer?.originVideoKey) {
|
||||
videoURL = `https://sns-video-bd.xhscdn.com/${video.consumer.originVideoKey}`;
|
||||
} else {
|
||||
const h264Streams = video.media?.stream?.h264;
|
||||
|
||||
if (h264Streams?.length) {
|
||||
videoURL = h264Streams.reduce((a, b) => Number(a?.videoBitrate) > Number(b?.videoBitrate) ? a : b).masterUrl;
|
||||
}
|
||||
}
|
||||
|
||||
if (!videoURL) return { error: "fetch.empty" };
|
||||
|
||||
return {
|
||||
urls: https(videoURL),
|
||||
filename: videoFilename,
|
||||
audioFilename: audioFilename,
|
||||
}
|
||||
}
|
||||
|
||||
if (!images || images.length === 0) {
|
||||
return { error: "fetch.empty" };
|
||||
}
|
||||
|
||||
if (images.length === 1) {
|
||||
return {
|
||||
isPhoto: true,
|
||||
urls: https(images[0].urlDefault),
|
||||
filename: `${filenameBase}.jpg`,
|
||||
}
|
||||
}
|
||||
|
||||
const picker = images.map((image, i) => {
|
||||
return {
|
||||
type: "photo",
|
||||
url: createStream({
|
||||
service: "xiaohongshu",
|
||||
type: "proxy",
|
||||
url: https(image.urlDefault),
|
||||
filename: `${filenameBase}_${i + 1}.jpg`,
|
||||
})
|
||||
}
|
||||
});
|
||||
|
||||
return { picker };
|
||||
}
|
|
@ -1,16 +1,16 @@
|
|||
import { fetch } from "undici";
|
||||
import HLS from "hls-parser";
|
||||
|
||||
import { fetch } from "undici";
|
||||
import { Innertube, Session } from "youtubei.js";
|
||||
|
||||
import { env } from "../../config.js";
|
||||
import { cleanString } from "../../misc/utils.js";
|
||||
import { getCookie, updateCookieValues } from "../cookie/manager.js";
|
||||
|
||||
const PLAYER_REFRESH_PERIOD = 1000 * 60 * 15; // ms
|
||||
|
||||
let innertube, lastRefreshedAt;
|
||||
|
||||
const codecMatch = {
|
||||
const codecList = {
|
||||
h264: {
|
||||
videoCodec: "avc1",
|
||||
audioCodec: "mp4a",
|
||||
|
@ -28,12 +28,27 @@ const codecMatch = {
|
|||
}
|
||||
}
|
||||
|
||||
const hlsCodecList = {
|
||||
h264: {
|
||||
videoCodec: "avc1",
|
||||
audioCodec: "mp4a",
|
||||
container: "mp4"
|
||||
},
|
||||
vp9: {
|
||||
videoCodec: "vp09",
|
||||
audioCodec: "mp4a",
|
||||
container: "webm"
|
||||
}
|
||||
}
|
||||
|
||||
const videoQualities = [144, 240, 360, 480, 720, 1080, 1440, 2160, 4320];
|
||||
|
||||
const transformSessionData = (cookie) => {
|
||||
if (!cookie)
|
||||
return;
|
||||
|
||||
const values = { ...cookie.values() };
|
||||
const REQUIRED_VALUES = [ 'access_token', 'refresh_token' ];
|
||||
const REQUIRED_VALUES = ['access_token', 'refresh_token'];
|
||||
|
||||
if (REQUIRED_VALUES.some(x => typeof values[x] !== 'string')) {
|
||||
return;
|
||||
|
@ -51,9 +66,18 @@ const transformSessionData = (cookie) => {
|
|||
|
||||
const cloneInnertube = async (customFetch) => {
|
||||
const shouldRefreshPlayer = lastRefreshedAt + PLAYER_REFRESH_PERIOD < new Date();
|
||||
|
||||
const rawCookie = getCookie('youtube');
|
||||
const rawCookieValues = rawCookie?.values();
|
||||
const cookie = rawCookie?.toString();
|
||||
|
||||
if (!innertube || shouldRefreshPlayer) {
|
||||
innertube = await Innertube.create({
|
||||
fetch: customFetch
|
||||
fetch: customFetch,
|
||||
retrieve_player: !!cookie,
|
||||
cookie,
|
||||
po_token: rawCookieValues?.po_token,
|
||||
visitor_data: rawCookieValues?.visitor_data,
|
||||
});
|
||||
lastRefreshedAt = +new Date();
|
||||
}
|
||||
|
@ -64,30 +88,30 @@ const cloneInnertube = async (customFetch) => {
|
|||
innertube.session.api_version,
|
||||
innertube.session.account_index,
|
||||
innertube.session.player,
|
||||
undefined,
|
||||
cookie,
|
||||
customFetch ?? innertube.session.http.fetch,
|
||||
innertube.session.cache
|
||||
);
|
||||
|
||||
const cookie = getCookie('youtube_oauth');
|
||||
const oauthData = transformSessionData(cookie);
|
||||
const oauthCookie = getCookie('youtube_oauth');
|
||||
const oauthData = transformSessionData(oauthCookie);
|
||||
|
||||
if (!session.logged_in && oauthData) {
|
||||
await session.oauth.init(oauthData);
|
||||
session.logged_in = true;
|
||||
}
|
||||
|
||||
if (session.logged_in) {
|
||||
if (session.logged_in && oauthData) {
|
||||
if (session.oauth.shouldRefreshToken()) {
|
||||
await session.oauth.refreshAccessToken();
|
||||
}
|
||||
|
||||
const cookieValues = cookie.values();
|
||||
const cookieValues = oauthCookie.values();
|
||||
const oldExpiry = new Date(cookieValues.expiry_date);
|
||||
const newExpiry = new Date(session.oauth.oauth2_tokens.expiry_date);
|
||||
|
||||
if (oldExpiry.getTime() !== newExpiry.getTime()) {
|
||||
updateCookieValues(cookie, {
|
||||
updateCookieValues(oauthCookie, {
|
||||
...session.oauth.client_id,
|
||||
...session.oauth.oauth2_tokens,
|
||||
expiry_date: newExpiry.toISOString()
|
||||
|
@ -99,7 +123,7 @@ const cloneInnertube = async (customFetch) => {
|
|||
return yt;
|
||||
}
|
||||
|
||||
export default async function(o) {
|
||||
export default async function (o) {
|
||||
let yt;
|
||||
try {
|
||||
yt = await cloneInnertube(
|
||||
|
@ -108,7 +132,7 @@ export default async function(o) {
|
|||
dispatcher: o.dispatcher
|
||||
})
|
||||
);
|
||||
} catch(e) {
|
||||
} catch (e) {
|
||||
if (e.message?.endsWith("decipher algorithm")) {
|
||||
return { error: "youtube.decipher" }
|
||||
} else if (e.message?.includes("refresh access token")) {
|
||||
|
@ -116,29 +140,46 @@ export default async function(o) {
|
|||
} else throw e;
|
||||
}
|
||||
|
||||
const quality = o.quality === "max" ? "9000" : o.quality;
|
||||
const cookie = getCookie('youtube')?.toString();
|
||||
|
||||
let info, isDubbed,
|
||||
format = o.format || "h264";
|
||||
let useHLS = o.youtubeHLS;
|
||||
|
||||
function qual(i) {
|
||||
if (!i.quality_label) {
|
||||
return;
|
||||
}
|
||||
|
||||
return i.quality_label.split('p')[0].split('s')[0]
|
||||
// HLS playlists don't contain the av1 video format, at least with the iOS client
|
||||
if (useHLS && o.format === "av1") {
|
||||
useHLS = false;
|
||||
}
|
||||
|
||||
let innertubeClient = o.innertubeClient || "ANDROID";
|
||||
|
||||
if (cookie) {
|
||||
useHLS = false;
|
||||
innertubeClient = "WEB";
|
||||
}
|
||||
|
||||
if (useHLS) {
|
||||
innertubeClient = "IOS";
|
||||
}
|
||||
|
||||
let info;
|
||||
try {
|
||||
info = await yt.getBasicInfo(o.id, yt.session.logged_in ? 'ANDROID' : 'IOS');
|
||||
} catch(e) {
|
||||
if (e?.info?.reason === "This video is private") {
|
||||
return { error: "content.video.private" };
|
||||
} else if (e?.message === "This video is unavailable") {
|
||||
return { error: "content.video.unavailable" };
|
||||
} else {
|
||||
return { error: "fetch.fail" };
|
||||
info = await yt.getBasicInfo(o.id, innertubeClient);
|
||||
} catch (e) {
|
||||
if (e?.info) {
|
||||
const errorInfo = JSON.parse(e?.info);
|
||||
|
||||
if (errorInfo?.reason === "This video is private") {
|
||||
return { error: "content.video.private" };
|
||||
}
|
||||
if (["INVALID_ARGUMENT", "UNAUTHENTICATED"].includes(errorInfo?.error?.status)) {
|
||||
return { error: "youtube.api_error" };
|
||||
}
|
||||
}
|
||||
|
||||
if (e?.message === "This video is unavailable") {
|
||||
return { error: "content.video.unavailable" };
|
||||
}
|
||||
|
||||
return { error: "fetch.fail" };
|
||||
}
|
||||
|
||||
if (!info) return { error: "fetch.fail" };
|
||||
|
@ -146,37 +187,47 @@ export default async function(o) {
|
|||
const playability = info.playability_status;
|
||||
const basicInfo = info.basic_info;
|
||||
|
||||
if (playability.status === "LOGIN_REQUIRED") {
|
||||
if (playability.reason.endsWith("bot")) {
|
||||
return { error: "youtube.login" }
|
||||
}
|
||||
if (playability.reason.endsWith("age")) {
|
||||
return { error: "content.video.age" }
|
||||
}
|
||||
if (playability?.error_screen?.reason?.text === "Private video") {
|
||||
return { error: "content.video.private" }
|
||||
}
|
||||
}
|
||||
switch (playability.status) {
|
||||
case "LOGIN_REQUIRED":
|
||||
if (playability.reason.endsWith("bot")) {
|
||||
return { error: "youtube.login" }
|
||||
}
|
||||
if (playability.reason.endsWith("age")) {
|
||||
return { error: "content.video.age" }
|
||||
}
|
||||
if (playability?.error_screen?.reason?.text === "Private video") {
|
||||
return { error: "content.video.private" }
|
||||
}
|
||||
break;
|
||||
|
||||
if (playability.status === "UNPLAYABLE") {
|
||||
if (playability?.reason?.endsWith("request limit.")) {
|
||||
return { error: "fetch.rate" }
|
||||
}
|
||||
if (playability?.error_screen?.subreason?.text?.endsWith("in your country")) {
|
||||
return { error: "content.video.region" }
|
||||
}
|
||||
if (playability?.error_screen?.reason?.text === "Private video") {
|
||||
return { error: "content.video.private" }
|
||||
}
|
||||
case "UNPLAYABLE":
|
||||
if (playability?.reason?.endsWith("request limit.")) {
|
||||
return { error: "fetch.rate" }
|
||||
}
|
||||
if (playability?.error_screen?.subreason?.text?.endsWith("in your country")) {
|
||||
return { error: "content.video.region" }
|
||||
}
|
||||
if (playability?.error_screen?.reason?.text === "Private video") {
|
||||
return { error: "content.video.private" }
|
||||
}
|
||||
break;
|
||||
|
||||
case "AGE_VERIFICATION_REQUIRED":
|
||||
return { error: "content.video.age" };
|
||||
}
|
||||
|
||||
if (playability.status !== "OK") {
|
||||
return { error: "content.video.unavailable" };
|
||||
}
|
||||
|
||||
if (basicInfo.is_live) {
|
||||
return { error: "content.video.live" };
|
||||
}
|
||||
|
||||
if (basicInfo.duration > env.durationLimit) {
|
||||
return { error: "content.too_long" };
|
||||
}
|
||||
|
||||
// return a critical error if returned video is "Video Not Available"
|
||||
// or a similar stub by youtube
|
||||
if (basicInfo.id !== o.id) {
|
||||
|
@ -186,64 +237,204 @@ export default async function(o) {
|
|||
}
|
||||
}
|
||||
|
||||
const filterByCodec = (formats) =>
|
||||
formats
|
||||
.filter(e =>
|
||||
e.mime_type.includes(codecMatch[format].videoCodec)
|
||||
|| e.mime_type.includes(codecMatch[format].audioCodec)
|
||||
)
|
||||
.sort((a, b) => Number(b.bitrate) - Number(a.bitrate));
|
||||
const quality = o.quality === "max" ? 9000 : Number(o.quality);
|
||||
|
||||
let adaptive_formats = filterByCodec(info.streaming_data.adaptive_formats);
|
||||
|
||||
if (adaptive_formats.length === 0 && format === "vp9") {
|
||||
format = "h264"
|
||||
adaptive_formats = filterByCodec(info.streaming_data.adaptive_formats)
|
||||
const normalizeQuality = res => {
|
||||
const shortestSide = Math.min(res.height, res.width);
|
||||
return videoQualities.find(qual => qual >= shortestSide);
|
||||
}
|
||||
|
||||
let bestQuality;
|
||||
let video, audio, dubbedLanguage,
|
||||
codec = o.format || "h264", itag = o.itag;
|
||||
|
||||
const bestVideo = adaptive_formats.find(i => i.has_video && i.content_length);
|
||||
const hasAudio = adaptive_formats.find(i => i.has_audio && i.content_length);
|
||||
if (useHLS) {
|
||||
const hlsManifest = info.streaming_data.hls_manifest_url;
|
||||
|
||||
if (bestVideo) bestQuality = qual(bestVideo);
|
||||
if (!hlsManifest) {
|
||||
return { error: "youtube.no_hls_streams" };
|
||||
}
|
||||
|
||||
if ((!bestQuality && !o.isAudioOnly) || !hasAudio)
|
||||
return { error: "youtube.codec" };
|
||||
const fetchedHlsManifest = await fetch(hlsManifest, {
|
||||
dispatcher: o.dispatcher,
|
||||
}).then(r => {
|
||||
if (r.status === 200) {
|
||||
return r.text();
|
||||
} else {
|
||||
throw new Error("couldn't fetch the HLS playlist");
|
||||
}
|
||||
}).catch(() => { });
|
||||
|
||||
if (basicInfo.duration > env.durationLimit)
|
||||
return { error: "content.too_long" };
|
||||
if (!fetchedHlsManifest) {
|
||||
return { error: "youtube.no_hls_streams" };
|
||||
}
|
||||
|
||||
const checkBestAudio = (i) => (i.has_audio && !i.has_video);
|
||||
const variants = HLS.parse(fetchedHlsManifest).variants.sort(
|
||||
(a, b) => Number(b.bandwidth) - Number(a.bandwidth)
|
||||
);
|
||||
|
||||
let audio = adaptive_formats.find(i =>
|
||||
checkBestAudio(i) && i.is_original
|
||||
);
|
||||
if (!variants || variants.length === 0) {
|
||||
return { error: "youtube.no_hls_streams" };
|
||||
}
|
||||
|
||||
if (o.dubLang) {
|
||||
let dubbedAudio = adaptive_formats.find(i =>
|
||||
checkBestAudio(i)
|
||||
&& i.language === o.dubLang
|
||||
&& i.audio_track
|
||||
)
|
||||
const matchHlsCodec = codecs => (
|
||||
codecs.includes(hlsCodecList[codec].videoCodec)
|
||||
);
|
||||
|
||||
if (dubbedAudio && !dubbedAudio?.audio_track?.audio_is_default) {
|
||||
audio = dubbedAudio;
|
||||
isDubbed = true;
|
||||
const best = variants.find(i => matchHlsCodec(i.codecs));
|
||||
|
||||
const preferred = variants.find(i =>
|
||||
matchHlsCodec(i.codecs) && normalizeQuality(i.resolution) === quality
|
||||
);
|
||||
|
||||
let selected = preferred || best;
|
||||
|
||||
if (!selected) {
|
||||
codec = "h264";
|
||||
selected = variants.find(i => matchHlsCodec(i.codecs));
|
||||
}
|
||||
|
||||
if (!selected) {
|
||||
return { error: "youtube.no_matching_format" };
|
||||
}
|
||||
|
||||
audio = selected.audio.find(i => i.isDefault);
|
||||
|
||||
// some videos (mainly those with AI dubs) don't have any tracks marked as default
|
||||
// why? god knows, but we assume that a default track is marked as such in the title
|
||||
if (!audio) {
|
||||
audio = selected.audio.find(i => i.name.endsWith("- original"));
|
||||
}
|
||||
|
||||
if (o.dubLang) {
|
||||
const dubbedAudio = selected.audio.find(i =>
|
||||
i.language?.startsWith(o.dubLang)
|
||||
);
|
||||
|
||||
if (dubbedAudio && !dubbedAudio.isDefault) {
|
||||
dubbedLanguage = dubbedAudio.language;
|
||||
audio = dubbedAudio;
|
||||
}
|
||||
}
|
||||
|
||||
selected.audio = [];
|
||||
selected.subtitles = [];
|
||||
video = selected;
|
||||
} else {
|
||||
// i miss typescript so bad
|
||||
const sorted_formats = {
|
||||
h264: {
|
||||
video: [],
|
||||
audio: [],
|
||||
bestVideo: undefined,
|
||||
bestAudio: undefined,
|
||||
},
|
||||
vp9: {
|
||||
video: [],
|
||||
audio: [],
|
||||
bestVideo: undefined,
|
||||
bestAudio: undefined,
|
||||
},
|
||||
av1: {
|
||||
video: [],
|
||||
audio: [],
|
||||
bestVideo: undefined,
|
||||
bestAudio: undefined,
|
||||
},
|
||||
}
|
||||
|
||||
const checkFormat = (format, pCodec) => format.content_length &&
|
||||
(format.mime_type.includes(codecList[pCodec].videoCodec)
|
||||
|| format.mime_type.includes(codecList[pCodec].audioCodec));
|
||||
|
||||
// sort formats & weed out bad ones
|
||||
info.streaming_data.adaptive_formats.sort((a, b) =>
|
||||
Number(b.bitrate) - Number(a.bitrate)
|
||||
).forEach(format => {
|
||||
Object.keys(codecList).forEach(yCodec => {
|
||||
const matchingItag = slot => !itag?.[slot] || itag[slot] === format.itag;
|
||||
const sorted = sorted_formats[yCodec];
|
||||
const goodFormat = checkFormat(format, yCodec);
|
||||
if (!goodFormat) return;
|
||||
|
||||
if (format.has_video && matchingItag('video')) {
|
||||
sorted.video.push(format);
|
||||
if (!sorted.bestVideo)
|
||||
sorted.bestVideo = format;
|
||||
}
|
||||
|
||||
if (format.has_audio && matchingItag('audio')) {
|
||||
sorted.audio.push(format);
|
||||
if (!sorted.bestAudio)
|
||||
sorted.bestAudio = format;
|
||||
}
|
||||
})
|
||||
});
|
||||
|
||||
const noBestMedia = () => {
|
||||
const vid = sorted_formats[codec]?.bestVideo;
|
||||
const aud = sorted_formats[codec]?.bestAudio;
|
||||
return (!vid && !o.isAudioOnly) || (!aud && o.isAudioOnly)
|
||||
};
|
||||
|
||||
if (noBestMedia()) {
|
||||
if (codec === "av1") codec = "vp9";
|
||||
else if (codec === "vp9") codec = "av1";
|
||||
|
||||
// if there's no higher quality fallback, then use h264
|
||||
if (noBestMedia()) codec = "h264";
|
||||
}
|
||||
|
||||
// if there's no proper combo of av1, vp9, or h264, then give up
|
||||
if (noBestMedia()) {
|
||||
return { error: "youtube.no_matching_format" };
|
||||
}
|
||||
|
||||
audio = sorted_formats[codec].bestAudio;
|
||||
|
||||
if (audio?.audio_track && !audio?.audio_track?.audio_is_default) {
|
||||
audio = sorted_formats[codec].audio.find(i =>
|
||||
i?.audio_track?.audio_is_default
|
||||
);
|
||||
}
|
||||
|
||||
if (o.dubLang) {
|
||||
const dubbedAudio = sorted_formats[codec].audio.find(i =>
|
||||
i.language?.startsWith(o.dubLang) && i.audio_track
|
||||
);
|
||||
|
||||
if (dubbedAudio && !dubbedAudio?.audio_track?.audio_is_default) {
|
||||
audio = dubbedAudio;
|
||||
dubbedLanguage = dubbedAudio.language;
|
||||
}
|
||||
}
|
||||
|
||||
if (!o.isAudioOnly) {
|
||||
const qual = (i) => {
|
||||
return normalizeQuality({
|
||||
width: i.width,
|
||||
height: i.height,
|
||||
})
|
||||
}
|
||||
|
||||
const bestQuality = qual(sorted_formats[codec].bestVideo);
|
||||
const useBestQuality = quality >= bestQuality;
|
||||
|
||||
video = useBestQuality
|
||||
? sorted_formats[codec].bestVideo
|
||||
: sorted_formats[codec].video.find(i => qual(i) === quality);
|
||||
|
||||
if (!video) video = sorted_formats[codec].bestVideo;
|
||||
}
|
||||
}
|
||||
|
||||
if (!audio) {
|
||||
audio = adaptive_formats.find(i => checkBestAudio(i));
|
||||
}
|
||||
|
||||
let fileMetadata = {
|
||||
title: cleanString(basicInfo.title.trim()),
|
||||
artist: cleanString(basicInfo.author.replace("- Topic", "").trim()),
|
||||
const fileMetadata = {
|
||||
title: basicInfo.title.trim(),
|
||||
artist: basicInfo.author.replace("- Topic", "").trim()
|
||||
}
|
||||
|
||||
if (basicInfo?.short_description?.startsWith("Provided to YouTube by")) {
|
||||
let descItems = basicInfo.short_description.split("\n\n", 5);
|
||||
const descItems = basicInfo.short_description.split("\n\n", 5);
|
||||
|
||||
if (descItems.length === 5) {
|
||||
fileMetadata.album = descItems[2];
|
||||
fileMetadata.copyright = descItems[3];
|
||||
|
@ -253,61 +444,94 @@ export default async function(o) {
|
|||
}
|
||||
}
|
||||
|
||||
let filenameAttributes = {
|
||||
const filenameAttributes = {
|
||||
service: "youtube",
|
||||
id: o.id,
|
||||
title: fileMetadata.title,
|
||||
author: fileMetadata.artist,
|
||||
youtubeDubName: isDubbed ? o.dubLang : false
|
||||
youtubeDubName: dubbedLanguage || false,
|
||||
}
|
||||
|
||||
if (audio && o.isAudioOnly) return {
|
||||
type: "audio",
|
||||
isAudioOnly: true,
|
||||
urls: audio.decipher(yt.session.player),
|
||||
filenameAttributes: filenameAttributes,
|
||||
fileMetadata: fileMetadata,
|
||||
bestAudio: format === "h264" ? "m4a" : "opus"
|
||||
}
|
||||
itag = {
|
||||
video: video?.itag,
|
||||
audio: audio?.itag
|
||||
};
|
||||
|
||||
const matchingQuality = Number(quality) > Number(bestQuality) ? bestQuality : quality,
|
||||
checkSingle = i =>
|
||||
qual(i) === matchingQuality && i.mime_type.includes(codecMatch[format].videoCodec),
|
||||
checkRender = i =>
|
||||
qual(i) === matchingQuality && i.has_video && !i.has_audio;
|
||||
const originalRequest = {
|
||||
...o,
|
||||
dispatcher: undefined,
|
||||
itag,
|
||||
innertubeClient
|
||||
};
|
||||
|
||||
let match, type, urls;
|
||||
if (audio && o.isAudioOnly) {
|
||||
let bestAudio = codec === "h264" ? "m4a" : "opus";
|
||||
let urls = audio.url;
|
||||
|
||||
// prefer good premuxed videos if available
|
||||
if (!o.isAudioOnly && !o.isAudioMuted && format === "h264" && bestVideo.fps <= 30) {
|
||||
match = info.streaming_data.formats.find(checkSingle);
|
||||
type = "proxy";
|
||||
urls = match?.decipher(yt.session.player);
|
||||
}
|
||||
if (useHLS) {
|
||||
bestAudio = "mp3";
|
||||
urls = audio.uri;
|
||||
}
|
||||
|
||||
const video = adaptive_formats.find(checkRender);
|
||||
if (innertubeClient === "WEB" && innertube) {
|
||||
urls = audio.decipher(innertube.session.player);
|
||||
}
|
||||
|
||||
if (!match && video && audio) {
|
||||
match = video;
|
||||
type = "merge";
|
||||
urls = [
|
||||
video.decipher(yt.session.player),
|
||||
audio.decipher(yt.session.player)
|
||||
]
|
||||
}
|
||||
|
||||
if (match) {
|
||||
filenameAttributes.qualityLabel = match.quality_label;
|
||||
filenameAttributes.resolution = `${match.width}x${match.height}`;
|
||||
filenameAttributes.extension = codecMatch[format].container;
|
||||
filenameAttributes.youtubeFormat = format;
|
||||
return {
|
||||
type,
|
||||
type: "audio",
|
||||
isAudioOnly: true,
|
||||
urls,
|
||||
filenameAttributes,
|
||||
fileMetadata
|
||||
fileMetadata,
|
||||
bestAudio,
|
||||
isHLS: useHLS,
|
||||
originalRequest
|
||||
}
|
||||
}
|
||||
|
||||
return { error: "fetch.fail" }
|
||||
if (video && audio) {
|
||||
let resolution;
|
||||
|
||||
if (useHLS) {
|
||||
resolution = normalizeQuality(video.resolution);
|
||||
filenameAttributes.resolution = `${video.resolution.width}x${video.resolution.height}`;
|
||||
filenameAttributes.extension = hlsCodecList[codec].container;
|
||||
|
||||
video = video.uri;
|
||||
audio = audio.uri;
|
||||
} else {
|
||||
resolution = normalizeQuality({
|
||||
width: video.width,
|
||||
height: video.height,
|
||||
});
|
||||
|
||||
filenameAttributes.resolution = `${video.width}x${video.height}`;
|
||||
filenameAttributes.extension = codecList[codec].container;
|
||||
|
||||
if (innertubeClient === "WEB" && innertube) {
|
||||
video = video.decipher(innertube.session.player);
|
||||
audio = audio.decipher(innertube.session.player);
|
||||
} else {
|
||||
video = video.url;
|
||||
audio = audio.url;
|
||||
}
|
||||
}
|
||||
|
||||
filenameAttributes.qualityLabel = `${resolution}p`;
|
||||
filenameAttributes.youtubeFormat = codec;
|
||||
|
||||
return {
|
||||
type: "merge",
|
||||
urls: [
|
||||
video,
|
||||
audio,
|
||||
],
|
||||
filenameAttributes,
|
||||
fileMetadata,
|
||||
isHLS: useHLS,
|
||||
originalRequest
|
||||
}
|
||||
}
|
||||
|
||||
return { error: "youtube.no_matching_format" };
|
||||
}
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
import psl from "psl";
|
||||
import psl from "@imput/psl";
|
||||
import { strict as assert } from "node:assert";
|
||||
|
||||
import { env } from "../config.js";
|
||||
|
@ -42,7 +42,7 @@ function aliasURL(url) {
|
|||
case "fixvx":
|
||||
case "x":
|
||||
if (services.twitter.altDomains.includes(url.hostname)) {
|
||||
url.hostname = 'twitter.com'
|
||||
url.hostname = 'twitter.com';
|
||||
}
|
||||
break;
|
||||
|
||||
|
@ -85,9 +85,21 @@ function aliasURL(url) {
|
|||
url.hostname = 'instagram.com';
|
||||
}
|
||||
break;
|
||||
|
||||
case "vk":
|
||||
case "vkvideo":
|
||||
if (services.vk.altDomains.includes(url.hostname)) {
|
||||
url.hostname = 'vk.com';
|
||||
}
|
||||
break;
|
||||
|
||||
case "xhslink":
|
||||
if (url.hostname === 'xhslink.com' && parts.length === 3) {
|
||||
url = new URL(`https://www.xiaohongshu.com/a/${parts[2]}`);
|
||||
}
|
||||
}
|
||||
|
||||
return url
|
||||
return url;
|
||||
}
|
||||
|
||||
function cleanURL(url) {
|
||||
|
@ -107,36 +119,41 @@ function cleanURL(url) {
|
|||
break;
|
||||
case "vk":
|
||||
if (url.pathname.includes('/clip') && url.searchParams.get('z')) {
|
||||
limitQuery('z')
|
||||
limitQuery('z');
|
||||
}
|
||||
break;
|
||||
case "youtube":
|
||||
if (url.searchParams.get('v')) {
|
||||
limitQuery('v')
|
||||
limitQuery('v');
|
||||
}
|
||||
break;
|
||||
case "rutube":
|
||||
if (url.searchParams.get('p')) {
|
||||
limitQuery('p')
|
||||
limitQuery('p');
|
||||
}
|
||||
break;
|
||||
case "twitter":
|
||||
if (url.searchParams.get('post_id')) {
|
||||
limitQuery('post_id')
|
||||
limitQuery('post_id');
|
||||
}
|
||||
break;
|
||||
case "xiaohongshu":
|
||||
if (url.searchParams.get('xsec_token')) {
|
||||
limitQuery('xsec_token');
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
if (stripQuery) {
|
||||
url.search = ''
|
||||
url.search = '';
|
||||
}
|
||||
|
||||
url.username = url.password = url.port = url.hash = ''
|
||||
url.username = url.password = url.port = url.hash = '';
|
||||
|
||||
if (url.pathname.endsWith('/'))
|
||||
url.pathname = url.pathname.slice(0, -1);
|
||||
|
||||
return url
|
||||
return url;
|
||||
}
|
||||
|
||||
function getHostIfValid(url) {
|
||||
|
@ -174,6 +191,11 @@ export function extract(url) {
|
|||
}
|
||||
|
||||
if (!env.enabledServices.has(host)) {
|
||||
// show a different message when youtube is disabled on official instances
|
||||
// as it only happens when shit hits the fan
|
||||
if (new URL(env.apiURL).hostname.endsWith(".imput.net") && host === "youtube") {
|
||||
return { error: "youtube.temporary_disabled" };
|
||||
}
|
||||
return { error: "service.disabled" };
|
||||
}
|
||||
|
||||
|
|
|
@ -1,7 +1,8 @@
|
|||
import { env } from "../config.js";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { Yellow } from "../misc/console-text.js";
|
||||
import { Green, Yellow } from "../misc/console-text.js";
|
||||
import ip from "ipaddr.js";
|
||||
import * as cluster from "../misc/cluster.js";
|
||||
|
||||
// this function is a modified variation of code
|
||||
// from https://stackoverflow.com/a/32402438/14855621
|
||||
|
@ -99,7 +100,9 @@ const formatKeys = (keyData) => {
|
|||
if (data.ips) {
|
||||
formatted[key].ips = data.ips.map(addr => {
|
||||
if (ip.isValid(addr)) {
|
||||
return [ ip.parse(addr), 32 ];
|
||||
const parsed = ip.parse(addr);
|
||||
const range = parsed.kind() === 'ipv6' ? 128 : 32;
|
||||
return [ parsed, range ];
|
||||
}
|
||||
|
||||
return ip.parseCIDR(addr);
|
||||
|
@ -114,6 +117,10 @@ const formatKeys = (keyData) => {
|
|||
return formatted;
|
||||
}
|
||||
|
||||
const updateKeys = (newKeys) => {
|
||||
keys = formatKeys(newKeys);
|
||||
}
|
||||
|
||||
const loadKeys = async (source) => {
|
||||
let updated;
|
||||
if (source.protocol === 'file:') {
|
||||
|
@ -129,12 +136,19 @@ const loadKeys = async (source) => {
|
|||
}
|
||||
|
||||
validateKeys(updated);
|
||||
keys = formatKeys(updated);
|
||||
|
||||
cluster.broadcast({ api_keys: updated });
|
||||
|
||||
updateKeys(updated);
|
||||
}
|
||||
|
||||
const wrapLoad = (url) => {
|
||||
const wrapLoad = (url, initial = false) => {
|
||||
loadKeys(url)
|
||||
.then(() => {})
|
||||
.then(() => {
|
||||
if (initial) {
|
||||
console.log(`${Green('[✓]')} api keys loaded successfully!`)
|
||||
}
|
||||
})
|
||||
.catch((e) => {
|
||||
console.error(`${Yellow('[!]')} Failed loading API keys at ${new Date().toISOString()}.`);
|
||||
console.error('Error:', e);
|
||||
|
@ -198,8 +212,16 @@ export const validateAuthorization = (req) => {
|
|||
}
|
||||
|
||||
export const setup = (url) => {
|
||||
wrapLoad(url);
|
||||
if (env.keyReloadInterval > 0) {
|
||||
setInterval(() => wrapLoad(url), env.keyReloadInterval * 1000);
|
||||
if (cluster.isPrimary) {
|
||||
wrapLoad(url, true);
|
||||
if (env.keyReloadInterval > 0) {
|
||||
setInterval(() => wrapLoad(url), env.keyReloadInterval * 1000);
|
||||
}
|
||||
} else if (cluster.isWorker) {
|
||||
process.on('message', (message) => {
|
||||
if ('api_keys' in message) {
|
||||
updateKeys(message.api_keys);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
62
api/src/security/secrets.js
Normal file
62
api/src/security/secrets.js
Normal file
|
@ -0,0 +1,62 @@
|
|||
import cluster from "node:cluster";
|
||||
import { createHmac, randomBytes } from "node:crypto";
|
||||
|
||||
const generateSalt = () => {
|
||||
if (cluster.isPrimary)
|
||||
return randomBytes(64);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
let rateSalt = generateSalt();
|
||||
let streamSalt = generateSalt();
|
||||
|
||||
export const syncSecrets = () => {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (cluster.isPrimary) {
|
||||
let remaining = Object.values(cluster.workers).length;
|
||||
const handleReady = (worker, m) => {
|
||||
if (m.ready)
|
||||
worker.send({ rateSalt, streamSalt });
|
||||
|
||||
if (!--remaining)
|
||||
resolve();
|
||||
}
|
||||
|
||||
for (const worker of Object.values(cluster.workers)) {
|
||||
worker.once(
|
||||
'message',
|
||||
(m) => handleReady(worker, m)
|
||||
);
|
||||
}
|
||||
} else if (cluster.isWorker) {
|
||||
if (rateSalt || streamSalt)
|
||||
return reject();
|
||||
|
||||
process.send({ ready: true });
|
||||
process.once('message', (message) => {
|
||||
if (rateSalt || streamSalt)
|
||||
return reject();
|
||||
|
||||
if (message.rateSalt && message.streamSalt) {
|
||||
streamSalt = Buffer.from(message.streamSalt);
|
||||
rateSalt = Buffer.from(message.rateSalt);
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
} else reject();
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
export const hashHmac = (value, type) => {
|
||||
let salt;
|
||||
if (type === 'rate')
|
||||
salt = rateSalt;
|
||||
else if (type === 'stream')
|
||||
salt = streamSalt;
|
||||
else
|
||||
throw "unknown salt";
|
||||
|
||||
return createHmac("sha256", salt).update(value).digest();
|
||||
}
|
48
api/src/store/base-store.js
Normal file
48
api/src/store/base-store.js
Normal file
|
@ -0,0 +1,48 @@
|
|||
const _stores = new Set();
|
||||
|
||||
export class Store {
|
||||
id;
|
||||
|
||||
constructor(name) {
|
||||
name = name.toUpperCase();
|
||||
|
||||
if (_stores.has(name))
|
||||
throw `${name} store already exists`;
|
||||
_stores.add(name);
|
||||
|
||||
this.id = name;
|
||||
}
|
||||
|
||||
async _has(_key) { await Promise.reject("needs implementation"); }
|
||||
has(key) {
|
||||
if (typeof key !== 'string') {
|
||||
key = key.toString();
|
||||
}
|
||||
|
||||
return this._has(key);
|
||||
}
|
||||
|
||||
async _get(_key) { await Promise.reject("needs implementation"); }
|
||||
async get(key) {
|
||||
if (typeof key !== 'string') {
|
||||
key = key.toString();
|
||||
}
|
||||
|
||||
const val = await this._get(key);
|
||||
if (val === null)
|
||||
return null;
|
||||
|
||||
return val;
|
||||
}
|
||||
|
||||
async _set(_key, _val, _exp_sec = -1) { await Promise.reject("needs implementation") }
|
||||
set(key, val, exp_sec = -1) {
|
||||
if (typeof key !== 'string') {
|
||||
key = key.toString();
|
||||
}
|
||||
|
||||
exp_sec = Math.round(exp_sec);
|
||||
|
||||
return this._set(key, val, exp_sec);
|
||||
}
|
||||
};
|
77
api/src/store/memory-store.js
Normal file
77
api/src/store/memory-store.js
Normal file
|
@ -0,0 +1,77 @@
|
|||
import { MinPriorityQueue } from '@datastructures-js/priority-queue';
|
||||
import { Store } from './base-store.js';
|
||||
|
||||
// minimum delay between sweeps to avoid repeatedly
|
||||
// sweeping entries close in proximity one by one.
|
||||
const MIN_THRESHOLD_MS = 2500;
|
||||
|
||||
export default class MemoryStore extends Store {
|
||||
#store = new Map();
|
||||
#timeouts = new MinPriorityQueue/*<{ t: number, k: unknown }>*/((obj) => obj.t);
|
||||
#nextSweep = { id: null, t: null };
|
||||
|
||||
constructor(name) {
|
||||
super(name);
|
||||
}
|
||||
|
||||
_has(key) {
|
||||
return this.#store.has(key);
|
||||
}
|
||||
|
||||
_get(key) {
|
||||
const val = this.#store.get(key);
|
||||
|
||||
return val === undefined ? null : val;
|
||||
}
|
||||
|
||||
_set(key, val, exp_sec = -1) {
|
||||
if (this.#store.has(key)) {
|
||||
this.#timeouts.remove(o => o.k === key);
|
||||
}
|
||||
|
||||
if (exp_sec > 0) {
|
||||
const exp = 1000 * exp_sec;
|
||||
const timeout_at = +new Date() + exp;
|
||||
|
||||
this.#timeouts.enqueue({ k: key, t: timeout_at });
|
||||
}
|
||||
|
||||
this.#store.set(key, val);
|
||||
this.#reschedule();
|
||||
}
|
||||
|
||||
#reschedule() {
|
||||
const current_time = new Date().getTime();
|
||||
const time = this.#timeouts.front()?.t;
|
||||
if (!time) {
|
||||
return;
|
||||
} else if (time < current_time) {
|
||||
return this.#sweepNow();
|
||||
}
|
||||
|
||||
const sweep = this.#nextSweep;
|
||||
if (sweep.id === null || sweep.t > time) {
|
||||
if (sweep.id) {
|
||||
clearTimeout(sweep.id);
|
||||
}
|
||||
|
||||
sweep.t = time;
|
||||
sweep.id = setTimeout(
|
||||
() => this.#sweepNow(),
|
||||
Math.max(MIN_THRESHOLD_MS, time - current_time)
|
||||
);
|
||||
sweep.id.unref();
|
||||
}
|
||||
}
|
||||
|
||||
#sweepNow() {
|
||||
while (this.#timeouts.front()?.t < new Date().getTime()) {
|
||||
const item = this.#timeouts.dequeue();
|
||||
this.#store.delete(item.k);
|
||||
}
|
||||
|
||||
this.#nextSweep.id = null;
|
||||
this.#nextSweep.t = null;
|
||||
this.#reschedule();
|
||||
}
|
||||
}
|
19
api/src/store/redis-ratelimit.js
Normal file
19
api/src/store/redis-ratelimit.js
Normal file
|
@ -0,0 +1,19 @@
|
|||
import { env } from "../config.js";
|
||||
|
||||
let client, redis, redisLimiter;
|
||||
|
||||
export const createStore = async (name) => {
|
||||
if (!env.redisURL) return;
|
||||
|
||||
if (!client) {
|
||||
redis = await import('redis');
|
||||
redisLimiter = await import('rate-limit-redis');
|
||||
client = redis.createClient({ url: env.redisURL });
|
||||
await client.connect();
|
||||
}
|
||||
|
||||
return new redisLimiter.default({
|
||||
prefix: `RL${name}_`,
|
||||
sendCommand: (...args) => client.sendCommand(args),
|
||||
});
|
||||
}
|
64
api/src/store/redis-store.js
Normal file
64
api/src/store/redis-store.js
Normal file
|
@ -0,0 +1,64 @@
|
|||
import { commandOptions, createClient } from "redis";
|
||||
import { env } from "../config.js";
|
||||
import { Store } from "./base-store.js";
|
||||
|
||||
export default class RedisStore extends Store {
|
||||
#client = createClient({
|
||||
url: env.redisURL,
|
||||
});
|
||||
#connected;
|
||||
|
||||
constructor(name) {
|
||||
super(name);
|
||||
this.#connected = this.#client.connect();
|
||||
}
|
||||
|
||||
#keyOf(key) {
|
||||
return this.id + '_' + key;
|
||||
}
|
||||
|
||||
async _has(key) {
|
||||
await this.#connected;
|
||||
|
||||
return this.#client.hExists(key);
|
||||
}
|
||||
|
||||
async _get(key) {
|
||||
await this.#connected;
|
||||
|
||||
const valueType = await this.#client.get(this.#keyOf(key) + '_t');
|
||||
const value = await this.#client.get(
|
||||
commandOptions({ returnBuffers: true }),
|
||||
this.#keyOf(key)
|
||||
);
|
||||
|
||||
if (!value) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (valueType === 'b')
|
||||
return value;
|
||||
else
|
||||
return JSON.parse(value);
|
||||
}
|
||||
|
||||
async _set(key, val, exp_sec = -1) {
|
||||
await this.#connected;
|
||||
|
||||
const options = exp_sec > 0 ? { EX: exp_sec } : undefined;
|
||||
|
||||
if (val instanceof Buffer) {
|
||||
await this.#client.set(
|
||||
this.#keyOf(key) + '_t',
|
||||
'b',
|
||||
options
|
||||
);
|
||||
}
|
||||
|
||||
await this.#client.set(
|
||||
this.#keyOf(key),
|
||||
val,
|
||||
options
|
||||
);
|
||||
}
|
||||
}
|
10
api/src/store/store.js
Normal file
10
api/src/store/store.js
Normal file
|
@ -0,0 +1,10 @@
|
|||
import { env } from '../config.js';
|
||||
|
||||
let _export;
|
||||
if (env.redisURL) {
|
||||
_export = await import('./redis-store.js');
|
||||
} else {
|
||||
_export = await import('./memory-store.js');
|
||||
}
|
||||
|
||||
export default _export.default;
|
|
@ -16,15 +16,17 @@ function transformObject(streamInfo, hlsObject) {
|
|||
|
||||
let fullUrl;
|
||||
if (getURL(hlsObject.uri)) {
|
||||
fullUrl = hlsObject.uri;
|
||||
fullUrl = new URL(hlsObject.uri);
|
||||
} else {
|
||||
fullUrl = new URL(hlsObject.uri, streamInfo.url);
|
||||
}
|
||||
|
||||
hlsObject.uri = createInternalStream(fullUrl.toString(), streamInfo);
|
||||
if (fullUrl.hostname !== '127.0.0.1') {
|
||||
hlsObject.uri = createInternalStream(fullUrl.toString(), streamInfo);
|
||||
|
||||
if (hlsObject.map) {
|
||||
hlsObject.map = transformObject(streamInfo, hlsObject.map);
|
||||
if (hlsObject.map) {
|
||||
hlsObject.map = transformObject(streamInfo, hlsObject.map);
|
||||
}
|
||||
}
|
||||
|
||||
return hlsObject;
|
||||
|
@ -53,7 +55,7 @@ function transformMediaPlaylist(streamInfo, hlsPlaylist) {
|
|||
|
||||
const HLS_MIME_TYPES = ["application/vnd.apple.mpegurl", "audio/mpegurl", "application/x-mpegURL"];
|
||||
|
||||
export function isHlsRequest (req) {
|
||||
export function isHlsResponse (req) {
|
||||
return HLS_MIME_TYPES.includes(req.headers['content-type']);
|
||||
}
|
||||
|
||||
|
|
|
@ -1,13 +1,13 @@
|
|||
import { request } from "undici";
|
||||
import { Readable } from "node:stream";
|
||||
import { closeRequest, getHeaders, pipe } from "./shared.js";
|
||||
import { handleHlsPlaylist, isHlsRequest } from "./internal-hls.js";
|
||||
import { handleHlsPlaylist, isHlsResponse } from "./internal-hls.js";
|
||||
|
||||
const CHUNK_SIZE = BigInt(8e6); // 8 MB
|
||||
const min = (a, b) => a < b ? a : b;
|
||||
|
||||
async function* readChunks(streamInfo, size) {
|
||||
let read = 0n;
|
||||
let read = 0n, chunksSinceTransplant = 0;
|
||||
while (read < size) {
|
||||
if (streamInfo.controller.signal.aborted) {
|
||||
throw new Error("controller aborted");
|
||||
|
@ -22,6 +22,16 @@ async function* readChunks(streamInfo, size) {
|
|||
signal: streamInfo.controller.signal
|
||||
});
|
||||
|
||||
if (chunk.statusCode === 403 && chunksSinceTransplant >= 3 && streamInfo.transplant) {
|
||||
chunksSinceTransplant = 0;
|
||||
try {
|
||||
await streamInfo.transplant(streamInfo.dispatcher);
|
||||
continue;
|
||||
} catch {}
|
||||
}
|
||||
|
||||
chunksSinceTransplant++;
|
||||
|
||||
const expected = min(CHUNK_SIZE, size - read);
|
||||
const received = BigInt(chunk.headers['content-length']);
|
||||
|
||||
|
@ -83,7 +93,7 @@ async function handleGenericStream(streamInfo, res) {
|
|||
const cleanup = () => res.end();
|
||||
|
||||
try {
|
||||
const req = await request(streamInfo.url, {
|
||||
const fileResponse = await request(streamInfo.url, {
|
||||
headers: {
|
||||
...Object.fromEntries(streamInfo.headers),
|
||||
host: undefined
|
||||
|
@ -93,19 +103,28 @@ async function handleGenericStream(streamInfo, res) {
|
|||
maxRedirections: 16
|
||||
});
|
||||
|
||||
res.status(req.statusCode);
|
||||
req.body.on('error', () => {});
|
||||
res.status(fileResponse.statusCode);
|
||||
fileResponse.body.on('error', () => {});
|
||||
|
||||
for (const [ name, value ] of Object.entries(req.headers))
|
||||
res.setHeader(name, value)
|
||||
// bluesky's cdn responds with wrong content-type for the hls playlist,
|
||||
// so we enforce it here until they fix it
|
||||
const isHls = isHlsResponse(fileResponse)
|
||||
|| (streamInfo.service === "bsky" && streamInfo.url.endsWith('.m3u8'));
|
||||
|
||||
if (req.statusCode < 200 || req.statusCode > 299)
|
||||
for (const [ name, value ] of Object.entries(fileResponse.headers)) {
|
||||
if (!isHls || name.toLowerCase() !== 'content-length') {
|
||||
res.setHeader(name, value);
|
||||
}
|
||||
}
|
||||
|
||||
if (fileResponse.statusCode < 200 || fileResponse.statusCode > 299) {
|
||||
return cleanup();
|
||||
}
|
||||
|
||||
if (isHlsRequest(req)) {
|
||||
await handleHlsPlaylist(streamInfo, req, res);
|
||||
if (isHls) {
|
||||
await handleHlsPlaylist(streamInfo, fileResponse, res);
|
||||
} else {
|
||||
pipe(req.body, res, cleanup);
|
||||
pipe(fileResponse.body, res, cleanup);
|
||||
}
|
||||
} catch {
|
||||
closeRequest(streamInfo.controller);
|
||||
|
@ -114,7 +133,11 @@ async function handleGenericStream(streamInfo, res) {
|
|||
}
|
||||
|
||||
export function internalStream(streamInfo, res) {
|
||||
if (streamInfo.service === 'youtube') {
|
||||
if (streamInfo.headers) {
|
||||
streamInfo.headers.delete('icy-metadata');
|
||||
}
|
||||
|
||||
if (streamInfo.service === 'youtube' && !streamInfo.isHLS) {
|
||||
return handleYoutubeStream(streamInfo, res);
|
||||
}
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
import NodeCache from "node-cache";
|
||||
import Store from "../store/store.js";
|
||||
|
||||
import { nanoid } from "nanoid";
|
||||
import { randomBytes } from "crypto";
|
||||
|
@ -7,34 +7,27 @@ import { setMaxListeners } from "node:events";
|
|||
|
||||
import { env } from "../config.js";
|
||||
import { closeRequest } from "./shared.js";
|
||||
import { decryptStream, encryptStream, generateHmac } from "../misc/crypto.js";
|
||||
import { decryptStream, encryptStream } from "../misc/crypto.js";
|
||||
import { hashHmac } from "../security/secrets.js";
|
||||
import { zip } from "../misc/utils.js";
|
||||
|
||||
// optional dependency
|
||||
const freebind = env.freebindCIDR && await import('freebind').catch(() => {});
|
||||
|
||||
const streamCache = new NodeCache({
|
||||
stdTTL: env.streamLifespan,
|
||||
checkperiod: 10,
|
||||
deleteOnExpire: true
|
||||
})
|
||||
|
||||
streamCache.on("expired", (key) => {
|
||||
streamCache.del(key);
|
||||
})
|
||||
const streamCache = new Store('streams');
|
||||
|
||||
const internalStreamCache = new Map();
|
||||
const hmacSalt = randomBytes(64).toString('hex');
|
||||
|
||||
export function createStream(obj) {
|
||||
const streamID = nanoid(),
|
||||
iv = randomBytes(16).toString('base64url'),
|
||||
secret = randomBytes(32).toString('base64url'),
|
||||
exp = new Date().getTime() + env.streamLifespan * 1000,
|
||||
hmac = generateHmac(`${streamID},${exp},${iv},${secret}`, hmacSalt),
|
||||
hmac = hashHmac(`${streamID},${exp},${iv},${secret}`, 'stream').toString('base64url'),
|
||||
streamData = {
|
||||
exp: exp,
|
||||
type: obj.type,
|
||||
urls: obj.u,
|
||||
urls: obj.url,
|
||||
service: obj.service,
|
||||
filename: obj.filename,
|
||||
|
||||
|
@ -46,12 +39,19 @@ export function createStream(obj) {
|
|||
audioBitrate: obj.audioBitrate,
|
||||
audioCopy: !!obj.audioCopy,
|
||||
audioFormat: obj.audioFormat,
|
||||
|
||||
isHLS: obj.isHLS || false,
|
||||
originalRequest: obj.originalRequest
|
||||
};
|
||||
|
||||
// FIXME: this is now a Promise, but it is not awaited
|
||||
// here. it may happen that the stream is not
|
||||
// stored in the Store before it is requested.
|
||||
streamCache.set(
|
||||
streamID,
|
||||
encryptStream(streamData, iv, secret)
|
||||
)
|
||||
encryptStream(streamData, iv, secret),
|
||||
env.streamLifespan
|
||||
);
|
||||
|
||||
let streamLink = new URL('/tunnel', env.apiURL);
|
||||
|
||||
|
@ -77,7 +77,7 @@ export function getInternalStream(id) {
|
|||
export function createInternalStream(url, obj = {}) {
|
||||
assert(typeof url === 'string');
|
||||
|
||||
let dispatcher;
|
||||
let dispatcher = obj.dispatcher;
|
||||
if (obj.requestIP) {
|
||||
dispatcher = freebind?.dispatcherFromIP(obj.requestIP, { strict: false })
|
||||
}
|
||||
|
@ -100,10 +100,12 @@ export function createInternalStream(url, obj = {}) {
|
|||
service: obj.service,
|
||||
headers,
|
||||
controller,
|
||||
dispatcher
|
||||
dispatcher,
|
||||
isHLS: obj.isHLS,
|
||||
transplant: obj.transplant
|
||||
});
|
||||
|
||||
let streamLink = new URL('/itunnel', `http://127.0.0.1:${env.apiPort}`);
|
||||
let streamLink = new URL('/itunnel', `http://127.0.0.1:${env.tunnelPort}`);
|
||||
streamLink.searchParams.set('id', streamID);
|
||||
|
||||
const cleanup = () => {
|
||||
|
@ -116,13 +118,17 @@ export function createInternalStream(url, obj = {}) {
|
|||
return streamLink.toString();
|
||||
}
|
||||
|
||||
export function destroyInternalStream(url) {
|
||||
function getInternalTunnelId(url) {
|
||||
url = new URL(url);
|
||||
if (url.hostname !== '127.0.0.1') {
|
||||
return;
|
||||
}
|
||||
|
||||
const id = url.searchParams.get('id');
|
||||
return url.searchParams.get('id');
|
||||
}
|
||||
|
||||
export function destroyInternalStream(url) {
|
||||
const id = getInternalTunnelId(url);
|
||||
|
||||
if (internalStreamCache.has(id)) {
|
||||
closeRequest(getInternalStream(id)?.controller);
|
||||
|
@ -130,9 +136,68 @@ export function destroyInternalStream(url) {
|
|||
}
|
||||
}
|
||||
|
||||
const transplantInternalTunnels = function(tunnelUrls, transplantUrls) {
|
||||
if (tunnelUrls.length !== transplantUrls.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const [ tun, url ] of zip(tunnelUrls, transplantUrls)) {
|
||||
const id = getInternalTunnelId(tun);
|
||||
const itunnel = getInternalStream(id);
|
||||
|
||||
if (!itunnel) continue;
|
||||
itunnel.url = url;
|
||||
}
|
||||
}
|
||||
|
||||
const transplantTunnel = async function (dispatcher) {
|
||||
if (this.pendingTransplant) {
|
||||
await this.pendingTransplant;
|
||||
return;
|
||||
}
|
||||
|
||||
let finished;
|
||||
this.pendingTransplant = new Promise(r => finished = r);
|
||||
|
||||
try {
|
||||
const handler = await import(`../processing/services/${this.service}.js`);
|
||||
const response = await handler.default({
|
||||
...this.originalRequest,
|
||||
dispatcher
|
||||
});
|
||||
|
||||
if (!response.urls) {
|
||||
return;
|
||||
}
|
||||
|
||||
response.urls = [response.urls].flat();
|
||||
if (this.originalRequest.isAudioOnly && response.urls.length > 1) {
|
||||
response.urls = [response.urls[1]];
|
||||
} else if (this.originalRequest.isAudioMuted) {
|
||||
response.urls = [response.urls[0]];
|
||||
}
|
||||
|
||||
const tunnels = [this.urls].flat();
|
||||
if (tunnels.length !== response.urls.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
transplantInternalTunnels(tunnels, response.urls);
|
||||
}
|
||||
catch {}
|
||||
finally {
|
||||
finished();
|
||||
delete this.pendingTransplant;
|
||||
}
|
||||
}
|
||||
|
||||
function wrapStream(streamInfo) {
|
||||
const url = streamInfo.urls;
|
||||
|
||||
if (streamInfo.originalRequest) {
|
||||
streamInfo.transplant = transplantTunnel.bind(streamInfo);
|
||||
}
|
||||
|
||||
if (typeof url === 'string') {
|
||||
streamInfo.urls = createInternalStream(url, streamInfo);
|
||||
} else if (Array.isArray(url)) {
|
||||
|
@ -146,10 +211,10 @@ function wrapStream(streamInfo) {
|
|||
return streamInfo;
|
||||
}
|
||||
|
||||
export function verifyStream(id, hmac, exp, secret, iv) {
|
||||
export async function verifyStream(id, hmac, exp, secret, iv) {
|
||||
try {
|
||||
const ghmac = generateHmac(`${id},${exp},${iv},${secret}`, hmacSalt);
|
||||
const cache = streamCache.get(id.toString());
|
||||
const ghmac = hashHmac(`${id},${exp},${iv},${secret}`, 'stream').toString('base64url');
|
||||
const cache = await streamCache.get(id.toString());
|
||||
|
||||
if (ghmac !== String(hmac)) return { status: 401 };
|
||||
if (!cache) return { status: 404 };
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
import { genericUserAgent } from "../config.js";
|
||||
import { vkClientAgent } from "../processing/services/vk.js";
|
||||
|
||||
const defaultHeaders = {
|
||||
'user-agent': genericUserAgent
|
||||
|
@ -13,6 +14,9 @@ const serviceHeaders = {
|
|||
origin: 'https://www.youtube.com',
|
||||
referer: 'https://www.youtube.com',
|
||||
DNT: '?1'
|
||||
},
|
||||
vk: {
|
||||
'user-agent': vkClientAgent
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -10,7 +10,7 @@ export default async function(res, streamInfo) {
|
|||
return await stream.proxy(streamInfo, res);
|
||||
|
||||
case "internal":
|
||||
return internalStream(streamInfo, res);
|
||||
return internalStream(streamInfo.data, res);
|
||||
|
||||
case "merge":
|
||||
return stream.merge(streamInfo, res);
|
||||
|
|
|
@ -1,10 +1,9 @@
|
|||
import { request } from "undici";
|
||||
import { Agent, request } from "undici";
|
||||
import ffmpeg from "ffmpeg-static";
|
||||
import { spawn } from "child_process";
|
||||
import { create as contentDisposition } from "content-disposition-header";
|
||||
|
||||
import { env } from "../config.js";
|
||||
import { metadataManager } from "../misc/utils.js";
|
||||
import { destroyInternalStream } from "./manage.js";
|
||||
import { hlsExceptions } from "../processing/service-config.js";
|
||||
import { getHeaders, closeRequest, closeResponse, pipe } from "./shared.js";
|
||||
|
@ -16,6 +15,29 @@ const ffmpegArgs = {
|
|||
gif: ["-vf", "scale=-1:-1:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse", "-loop", "0"]
|
||||
}
|
||||
|
||||
const metadataTags = [
|
||||
"album",
|
||||
"copyright",
|
||||
"title",
|
||||
"artist",
|
||||
"track",
|
||||
"date",
|
||||
];
|
||||
|
||||
const convertMetadataToFFmpeg = (metadata) => {
|
||||
let args = [];
|
||||
|
||||
for (const [ name, value ] of Object.entries(metadata)) {
|
||||
if (metadataTags.includes(name)) {
|
||||
args.push('-metadata', `${name}=${value.replace(/[\u0000-\u0009]/g, "")}`);
|
||||
} else {
|
||||
throw `${name} metadata tag is not supported.`;
|
||||
}
|
||||
}
|
||||
|
||||
return args;
|
||||
}
|
||||
|
||||
const toRawHeaders = (headers) => {
|
||||
return Object.entries(headers)
|
||||
.map(([key, value]) => `${key}: ${value}\r\n`)
|
||||
|
@ -38,6 +60,8 @@ const getCommand = (args) => {
|
|||
return [ffmpeg, args]
|
||||
}
|
||||
|
||||
const defaultAgent = new Agent();
|
||||
|
||||
const proxy = async (streamInfo, res) => {
|
||||
const abortController = new AbortController();
|
||||
const shutdown = () => (
|
||||
|
@ -56,7 +80,8 @@ const proxy = async (streamInfo, res) => {
|
|||
Range: streamInfo.range
|
||||
},
|
||||
signal: abortController.signal,
|
||||
maxRedirections: 16
|
||||
maxRedirections: 16,
|
||||
dispatcher: defaultAgent,
|
||||
});
|
||||
|
||||
res.status(statusCode);
|
||||
|
@ -101,12 +126,16 @@ const merge = (streamInfo, res) => {
|
|||
|
||||
args = args.concat(ffmpegArgs[format]);
|
||||
|
||||
if (hlsExceptions.includes(streamInfo.service)) {
|
||||
args.push('-bsf:a', 'aac_adtstoasc')
|
||||
if (hlsExceptions.includes(streamInfo.service) && streamInfo.isHLS) {
|
||||
if (streamInfo.service === "youtube" && format === "webm") {
|
||||
args.push('-c:a', 'libopus');
|
||||
} else {
|
||||
args.push('-c:a', 'aac', '-bsf:a', 'aac_adtstoasc');
|
||||
}
|
||||
}
|
||||
|
||||
if (streamInfo.metadata) {
|
||||
args = args.concat(metadataManager(streamInfo.metadata))
|
||||
args = args.concat(convertMetadataToFFmpeg(streamInfo.metadata))
|
||||
}
|
||||
|
||||
args.push('-f', format, 'pipe:3');
|
||||
|
@ -238,7 +267,7 @@ const convertAudio = (streamInfo, res) => {
|
|||
}
|
||||
|
||||
if (streamInfo.metadata) {
|
||||
args = args.concat(metadataManager(streamInfo.metadata))
|
||||
args = args.concat(convertMetadataToFFmpeg(streamInfo.metadata))
|
||||
}
|
||||
|
||||
args.push('-f', streamInfo.audioFormat === "m4a" ? "ipod" : streamInfo.audioFormat, 'pipe:3');
|
||||
|
|
22
api/src/util/generate-jwt-secret.js
Normal file
22
api/src/util/generate-jwt-secret.js
Normal file
|
@ -0,0 +1,22 @@
|
|||
// run with `pnpm -r token:jwt`
|
||||
|
||||
const makeSecureString = (length = 64) => {
|
||||
const alphabet = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-';
|
||||
const out = [];
|
||||
|
||||
while (out.length < length) {
|
||||
for (const byte of crypto.getRandomValues(new Uint8Array(length))) {
|
||||
if (byte < alphabet.length) {
|
||||
out.push(alphabet[byte]);
|
||||
}
|
||||
|
||||
if (out.length === length) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return out.join('');
|
||||
}
|
||||
|
||||
console.log(`JWT_SECRET: ${JSON.stringify(makeSecureString(64))}`)
|
|
@ -1,105 +0,0 @@
|
|||
import { existsSync, unlinkSync, appendFileSync } from "fs";
|
||||
import { createInterface } from "readline";
|
||||
import { Cyan, Bright } from "../misc/console-text.js";
|
||||
import { loadJSON } from "../misc/load-from-fs.js";
|
||||
import { execSync } from "child_process";
|
||||
|
||||
const { version } = loadJSON("./package.json");
|
||||
|
||||
let envPath = './.env';
|
||||
let q = `${Cyan('?')} \x1b[1m`;
|
||||
let ob = {};
|
||||
let rl = createInterface({ input: process.stdin, output: process.stdout });
|
||||
|
||||
let final = () => {
|
||||
if (existsSync(envPath)) unlinkSync(envPath);
|
||||
|
||||
for (let i in ob) {
|
||||
appendFileSync(envPath, `${i}=${ob[i]}\n`)
|
||||
}
|
||||
console.log(Bright("\nAwesome! I've created a fresh .env file for you."));
|
||||
console.log(`${Bright("Now I'll run")} ${Cyan("npm install")} ${Bright("to install all dependencies. It shouldn't take long.\n\n")}`);
|
||||
execSync('pnpm install', { stdio: [0, 1, 2] });
|
||||
console.log(`\n\n${Cyan("All done!\n")}`);
|
||||
console.log(Bright("You can re-run this script at any time to update the configuration."));
|
||||
console.log(Bright("\nYou're now ready to start cobalt. Simply run ") + Cyan("npm start") + Bright('!\nHave fun :)'));
|
||||
rl.close()
|
||||
}
|
||||
|
||||
console.log(
|
||||
`${Cyan(`Hey, this is cobalt v.${version}!`)}\n${Bright("Let's start by creating a new ")}${Cyan(".env")}${Bright(" file. You can always change it later.")}`
|
||||
)
|
||||
|
||||
function setup() {
|
||||
console.log(Bright("\nWhat kind of server will this instance be?\nOptions: api, web."));
|
||||
|
||||
rl.question(q, r1 => {
|
||||
switch (r1.toLowerCase()) {
|
||||
case 'api':
|
||||
console.log(Bright("\nCool! What's the domain this API instance will be running on? (localhost)\nExample: api.cobalt.tools"));
|
||||
|
||||
rl.question(q, apiURL => {
|
||||
ob.API_URL = `http://localhost:9000/`;
|
||||
ob.API_PORT = 9000;
|
||||
if (apiURL && apiURL !== "localhost") ob.API_URL = `https://${apiURL.toLowerCase()}/`;
|
||||
|
||||
console.log(Bright("\nGreat! Now, what port will it be running on? (9000)"));
|
||||
|
||||
rl.question(q, apiPort => {
|
||||
if (apiPort) ob.API_PORT = apiPort;
|
||||
if (apiPort && (apiURL === "localhost" || !apiURL)) ob.API_URL = `http://localhost:${apiPort}/`;
|
||||
|
||||
console.log(Bright("\nWhat will your instance's name be? Usually it's something like eu-nl aka region-country. (local)"));
|
||||
|
||||
rl.question(q, apiName => {
|
||||
ob.API_NAME = apiName.toLowerCase();
|
||||
if (!apiName || apiName === "local") ob.API_NAME = "local";
|
||||
|
||||
console.log(Bright("\nOne last thing: would you like to enable CORS? It allows other websites and extensions to use your instance's API.\ny/n (n)"));
|
||||
|
||||
rl.question(q, apiCors => {
|
||||
let answCors = apiCors.toLowerCase().trim();
|
||||
if (answCors !== "y" && answCors !== "yes") ob.CORS_WILDCARD = '0'
|
||||
final()
|
||||
})
|
||||
})
|
||||
});
|
||||
|
||||
})
|
||||
break;
|
||||
case 'web':
|
||||
console.log(Bright("\nAwesome! What's the domain this web app instance will be running on? (localhost)\nExample: cobalt.tools"));
|
||||
|
||||
rl.question(q, webURL => {
|
||||
ob.WEB_URL = `http://localhost:9001/`;
|
||||
ob.WEB_PORT = 9001;
|
||||
if (webURL && webURL !== "localhost") ob.WEB_URL = `https://${webURL.toLowerCase()}/`;
|
||||
|
||||
console.log(
|
||||
Bright("\nGreat! Now, what port will it be running on? (9001)")
|
||||
)
|
||||
rl.question(q, webPort => {
|
||||
if (webPort) ob.WEB_PORT = webPort;
|
||||
if (webPort && (webURL === "localhost" || !webURL)) ob.WEB_URL = `http://localhost:${webPort}/`;
|
||||
|
||||
console.log(
|
||||
Bright("\nOne last thing: what default API domain should be used? (api.cobalt.tools)\nIf it's hosted locally, make sure to include the port:") + Cyan(" localhost:9000")
|
||||
);
|
||||
|
||||
rl.question(q, apiURL => {
|
||||
ob.API_URL = `https://${apiURL.toLowerCase()}/`;
|
||||
if (apiURL.includes(':')) ob.API_URL = `http://${apiURL.toLowerCase()}/`;
|
||||
if (!apiURL) ob.API_URL = "https://api.cobalt.tools/";
|
||||
final()
|
||||
})
|
||||
});
|
||||
|
||||
});
|
||||
break;
|
||||
default:
|
||||
console.log(Bright("\nThis is not an option. Try again."));
|
||||
setup()
|
||||
}
|
||||
})
|
||||
}
|
||||
setup()
|
|
@ -1,82 +0,0 @@
|
|||
import { env } from "../config.js";
|
||||
import { runTest } from "../misc/run-test.js";
|
||||
import { loadJSON } from "../misc/load-from-fs.js";
|
||||
import { Red, Bright } from "../misc/console-text.js";
|
||||
import { randomizeCiphers } from "../misc/randomize-ciphers.js";
|
||||
|
||||
import { services } from "../processing/service-config.js";
|
||||
|
||||
const tests = loadJSON('./src/util/tests.json');
|
||||
|
||||
// services that are known to frequently fail due to external
|
||||
// factors (e.g. rate limiting)
|
||||
const finnicky = new Set(['bilibili', 'instagram', 'facebook', 'youtube'])
|
||||
|
||||
const action = process.argv[2];
|
||||
switch (action) {
|
||||
case "get-services":
|
||||
const fromConfig = Object.keys(services);
|
||||
|
||||
const missingTests = fromConfig.filter(
|
||||
service => !tests[service] || tests[service].length === 0
|
||||
);
|
||||
|
||||
if (missingTests.length) {
|
||||
console.error('services have no tests:', missingTests);
|
||||
console.log('[]');
|
||||
process.exitCode = 1;
|
||||
break;
|
||||
}
|
||||
|
||||
console.log(JSON.stringify(fromConfig));
|
||||
break;
|
||||
|
||||
case "run-tests-for":
|
||||
const service = process.argv[3];
|
||||
let failed = false;
|
||||
|
||||
if (!tests[service]) {
|
||||
console.error('no such service:', service);
|
||||
}
|
||||
|
||||
env.streamLifespan = 10000;
|
||||
env.apiURL = 'http://x';
|
||||
randomizeCiphers();
|
||||
|
||||
for (const test of tests[service]) {
|
||||
const { name, url, params, expected } = test;
|
||||
const canFail = test.canFail || finnicky.has(service);
|
||||
|
||||
try {
|
||||
await runTest(url, params, expected);
|
||||
console.log(`${service}/${name}: ok`);
|
||||
|
||||
} catch(e) {
|
||||
failed = !canFail;
|
||||
|
||||
let failText = canFail ? `${Red('FAIL')} (ignored)` : Bright(Red('FAIL'));
|
||||
if (canFail && process.env.GITHUB_ACTION) {
|
||||
console.log(`::warning title=${service}/${name.replace(/,/g, ';')}::failed and was ignored`);
|
||||
}
|
||||
|
||||
console.error(`${service}/${name}: ${failText}`);
|
||||
const errorString = e.toString().split('\n');
|
||||
let c = '┃';
|
||||
errorString.forEach((line, index) => {
|
||||
line = line.replace('!=', Red('!='));
|
||||
|
||||
if (index === errorString.length - 1) {
|
||||
c = '┗';
|
||||
}
|
||||
|
||||
console.error(` ${c}`, line);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
process.exitCode = Number(failed);
|
||||
break;
|
||||
default:
|
||||
console.error('invalid action:', action);
|
||||
process.exitCode = 1;
|
||||
}
|
|
@ -1,84 +1,129 @@
|
|||
import "dotenv/config";
|
||||
import path from "node:path";
|
||||
|
||||
import { env } from "../config.js";
|
||||
import { runTest } from "../misc/run-test.js";
|
||||
import { loadJSON } from "../misc/load-from-fs.js";
|
||||
import { Red, Bright } from "../misc/console-text.js";
|
||||
import { randomizeCiphers } from "../misc/randomize-ciphers.js";
|
||||
|
||||
import { services } from "../processing/service-config.js";
|
||||
import { extract } from "../processing/url.js";
|
||||
import match from "../processing/match.js";
|
||||
import { loadJSON } from "../misc/load-from-fs.js";
|
||||
import { normalizeRequest } from "../processing/request.js";
|
||||
import { env } from "../config.js";
|
||||
|
||||
env.apiURL = 'http://localhost:9000'
|
||||
let tests = loadJSON('./src/util/tests.json');
|
||||
const getTestPath = service => path.join('./src/util/tests/', `./${service}.json`);
|
||||
const getTests = (service) => loadJSON(getTestPath(service));
|
||||
|
||||
let noTest = [];
|
||||
let failed = [];
|
||||
let success = 0;
|
||||
// services that are known to frequently fail due to external
|
||||
// factors (e.g. rate limiting)
|
||||
const finnicky = new Set(['bilibili', 'instagram', 'facebook', 'youtube', 'vk', 'twitter']);
|
||||
|
||||
function addToFail(service, testName, url, status, response) {
|
||||
failed.push({
|
||||
service: service,
|
||||
name: testName,
|
||||
url: url,
|
||||
status: status,
|
||||
response: response
|
||||
})
|
||||
}
|
||||
for (let i in services) {
|
||||
if (tests[i]) {
|
||||
console.log(`\nRunning tests for ${i}...\n`)
|
||||
for (let k = 0; k < tests[i].length; k++) {
|
||||
let test = tests[i][k];
|
||||
const runTestsFor = async (service) => {
|
||||
const tests = getTests(service);
|
||||
let softFails = 0, fails = 0;
|
||||
|
||||
console.log(`Running test ${k+1}: ${test.name}`);
|
||||
console.log('params:');
|
||||
let params = {...{url: test.url}, ...test.params};
|
||||
console.log(params);
|
||||
|
||||
let chck = await normalizeRequest(params);
|
||||
if (chck.success) {
|
||||
chck = chck.data;
|
||||
|
||||
const parsed = extract(chck.url);
|
||||
if (parsed === null) {
|
||||
throw `Invalid URL: ${chck.url}`
|
||||
}
|
||||
|
||||
let j = await match({
|
||||
host: parsed.host,
|
||||
patternMatch: parsed.patternMatch,
|
||||
params: chck,
|
||||
});
|
||||
console.log('\nReceived:');
|
||||
console.log(j)
|
||||
if (j.status === test.expected.code && j.body.status === test.expected.status) {
|
||||
console.log("\n✅ Success.\n");
|
||||
success++
|
||||
} else {
|
||||
console.log(`\n❌ Fail. Expected: ${test.expected.code} & ${test.expected.status}, received: ${j.status} & ${j.body.status}\n`);
|
||||
addToFail(i, test.name, test.url, j.body.status, j)
|
||||
}
|
||||
} else {
|
||||
console.log("\n❌ couldn't validate the request JSON.\n");
|
||||
addToFail(i, test.name, test.url, "unknown", {})
|
||||
}
|
||||
}
|
||||
console.log("\n\n")
|
||||
} else {
|
||||
console.warn(`No tests found for ${i}.`);
|
||||
noTest.push(i)
|
||||
if (!tests) {
|
||||
throw "no such service: " + service;
|
||||
}
|
||||
|
||||
for (const test of tests) {
|
||||
const { name, url, params, expected } = test;
|
||||
const canFail = test.canFail || finnicky.has(service);
|
||||
|
||||
try {
|
||||
await runTest(url, params, expected);
|
||||
console.log(`${service}/${name}: ok`);
|
||||
|
||||
} catch (e) {
|
||||
softFails += !canFail;
|
||||
fails++;
|
||||
|
||||
let failText = canFail ? `${Red('FAIL')} (ignored)` : Bright(Red('FAIL'));
|
||||
if (canFail && process.env.GITHUB_ACTION) {
|
||||
console.log(`::warning title=${service}/${name.replace(/,/g, ';')}::failed and was ignored`);
|
||||
}
|
||||
|
||||
console.error(`${service}/${name}: ${failText}`);
|
||||
const errorString = e.toString().split('\n');
|
||||
let c = '┃';
|
||||
errorString.forEach((line, index) => {
|
||||
line = line.replace('!=', Red('!='));
|
||||
|
||||
if (index === errorString.length - 1) {
|
||||
c = '┗';
|
||||
}
|
||||
|
||||
console.error(` ${c}`, line);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return { fails, softFails };
|
||||
}
|
||||
|
||||
console.log(`✅ ${success} tests succeeded.`);
|
||||
console.log(`❌ ${failed.length} tests failed.`);
|
||||
console.log(`❔ ${noTest.length} services weren't tested.`);
|
||||
|
||||
if (failed.length > 0) {
|
||||
console.log(`\nFailed tests:`);
|
||||
console.log(failed)
|
||||
const printHeader = (service, padLen) => {
|
||||
const padding = padLen - service.length;
|
||||
service = service.padEnd(1 + service.length + padding, ' ');
|
||||
console.log(service + '='.repeat(50));
|
||||
}
|
||||
|
||||
if (noTest.length > 0) {
|
||||
console.log(`\nMissing tests:`);
|
||||
console.log(noTest)
|
||||
const action = process.argv[2];
|
||||
switch (action) {
|
||||
case "get-services":
|
||||
const fromConfig = Object.keys(services);
|
||||
|
||||
const missingTests = fromConfig.filter(
|
||||
service => {
|
||||
const tests = getTests(service);
|
||||
return !tests || tests.length === 0
|
||||
}
|
||||
);
|
||||
|
||||
if (missingTests.length) {
|
||||
console.error('services have no tests:', missingTests);
|
||||
process.exitCode = 1;
|
||||
break;
|
||||
}
|
||||
|
||||
console.log(JSON.stringify(fromConfig));
|
||||
break;
|
||||
|
||||
case "run-tests-for":
|
||||
env.streamLifespan = 10000;
|
||||
env.apiURL = 'http://x/';
|
||||
randomizeCiphers();
|
||||
|
||||
try {
|
||||
const { softFails } = await runTestsFor(process.argv[3]);
|
||||
process.exitCode = Number(!!softFails);
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
process.exitCode = 1;
|
||||
break;
|
||||
}
|
||||
|
||||
break;
|
||||
default:
|
||||
const maxHeaderLen = Object.keys(services).reduce((n, v) => v.length > n ? v.length : n, 0);
|
||||
const failCounters = {};
|
||||
|
||||
env.streamLifespan = 10000;
|
||||
env.apiURL = 'http://x/';
|
||||
randomizeCiphers();
|
||||
|
||||
for (const service in services) {
|
||||
printHeader(service, maxHeaderLen);
|
||||
const { fails, softFails } = await runTestsFor(service);
|
||||
failCounters[service] = fails;
|
||||
console.log();
|
||||
|
||||
if (!process.exitCode && softFails)
|
||||
process.exitCode = 1;
|
||||
}
|
||||
|
||||
console.log('='.repeat(50 + maxHeaderLen));
|
||||
console.log(
|
||||
Bright('total fails:'),
|
||||
Object.values(failCounters).reduce((a, b) => a + b)
|
||||
);
|
||||
for (const [ service, fails ] of Object.entries(failCounters)) {
|
||||
if (fails) console.log(`${Bright(service)} fails: ${fails}`);
|
||||
}
|
||||
}
|
||||
|
|
File diff suppressed because it is too large
Load diff
60
api/src/util/tests/bilibili.json
Normal file
60
api/src/util/tests/bilibili.json
Normal file
|
@ -0,0 +1,60 @@
|
|||
[
|
||||
{
|
||||
"name": "1080p video",
|
||||
"url": "https://www.bilibili.com/video/BV18i4y1m7xV/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1080p video muted",
|
||||
"url": "https://www.bilibili.com/video/BV18i4y1m7xV/",
|
||||
"params": {
|
||||
"downloadMode": "mute"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1080p vertical video",
|
||||
"url": "https://www.bilibili.com/video/BV1uu411z7VV/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1080p vertical video muted",
|
||||
"url": "https://www.bilibili.com/video/BV1uu411z7VV/",
|
||||
"params": {
|
||||
"downloadMode": "mute"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "b23.tv shortlink",
|
||||
"url": "https://b23.tv/lbMyOI9",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "bilibili.tv link",
|
||||
"url": "https://www.bilibili.tv/en/video/4789599404426256",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
}
|
||||
]
|
96
api/src/util/tests/bsky.json
Normal file
96
api/src/util/tests/bsky.json
Normal file
|
@ -0,0 +1,96 @@
|
|||
[
|
||||
{
|
||||
"name": "horizontal video",
|
||||
"url": "https://bsky.app/profile/did:plc:oisofpd7lj26yvgiivf3lxsi/post/3l3giwtwp222m",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "horizontal video, recordWithMedia",
|
||||
"url": "https://bsky.app/profile/did:plc:ywbm3iywnhzep3ckt6efhoh7/post/3l3wonhk23g2i",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "vertical video",
|
||||
"url": "https://bsky.app/profile/did:plc:oisofpd7lj26yvgiivf3lxsi/post/3l3jhpomhjk2m",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "vertical video (muted)",
|
||||
"url": "https://bsky.app/profile/did:plc:oisofpd7lj26yvgiivf3lxsi/post/3l3jhpomhjk2m",
|
||||
"params": {
|
||||
"downloadMode": "mute"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "vertical video (audio)",
|
||||
"url": "https://bsky.app/profile/did:plc:oisofpd7lj26yvgiivf3lxsi/post/3l3jhpomhjk2m",
|
||||
"params": {
|
||||
"downloadMode": "audio"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "single image",
|
||||
"url": "https://bsky.app/profile/did:plc:k4a7d65fcyevbrnntjxh57go/post/3l33flpoygt26",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "gif with a quoted post",
|
||||
"url": "https://bsky.app/profile/imlunahey.com/post/3lgajpn5dtk2t",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "gif alone in a post",
|
||||
"url": "https://bsky.app/profile/imlunahey.com/post/3lgah3ovxnc2q",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "several images",
|
||||
"url": "https://bsky.app/profile/did:plc:rai7s6su2sy22ss7skouedl7/post/3kzxuxbiul626",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "picker"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "deleted post/invalid user",
|
||||
"url": "https://bsky.app/profile/notreal.bsky.team/post/3l2udah76ch2c",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 400,
|
||||
"status": "error"
|
||||
}
|
||||
}
|
||||
]
|
29
api/src/util/tests/dailymotion.json
Normal file
29
api/src/util/tests/dailymotion.json
Normal file
|
@ -0,0 +1,29 @@
|
|||
[
|
||||
{
|
||||
"name": "regular video",
|
||||
"url": "https://www.dailymotion.com/video/x8t1eho",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "private video",
|
||||
"url": "https://www.dailymotion.com/video/k41fZWpx2TaAORA2nok",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "dai.ly shortened link",
|
||||
"url": "https://dai.ly/k41fZWpx2TaAORA2nok",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
}
|
||||
]
|
67
api/src/util/tests/facebook.json
Normal file
67
api/src/util/tests/facebook.json
Normal file
|
@ -0,0 +1,67 @@
|
|||
[
|
||||
{
|
||||
"name": "direct video with username and id",
|
||||
"url": "https://web.facebook.com/100048111287134/videos/1157798148685638/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "direct video with id as query param",
|
||||
"url": "https://web.facebook.com/watch/?v=883839773514682&ref=sharing",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "direct video with caption",
|
||||
"url": "https://web.facebook.com/wood57/videos/𝐒𝐞𝐛𝐚𝐬𝐤𝐨𝐦-𝐟𝐮𝐥𝐥/883839773514682",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "shortlink video",
|
||||
"url": "https://fb.watch/r1K6XHMfGT/",
|
||||
"canFail": true,
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "reel video",
|
||||
"url": "https://web.facebook.com/reel/730293269054758",
|
||||
"canFail": true,
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "shared video link",
|
||||
"url": "https://www.facebook.com/share/v/NEf87jbPTvFE8LsL/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "shared video link v2",
|
||||
"url": "https://web.facebook.com/share/r/JFZfPVgLkiJQmWrr/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
}
|
||||
]
|
123
api/src/util/tests/instagram.json
Normal file
123
api/src/util/tests/instagram.json
Normal file
|
@ -0,0 +1,123 @@
|
|||
[
|
||||
{
|
||||
"name": "single photo post",
|
||||
"url": "https://www.instagram.com/p/CwIgW8Yu5-I/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "various picker (photos + video)",
|
||||
"url": "https://www.instagram.com/p/CvYrSgnsKjv/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "picker"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "reel",
|
||||
"url": "https://www.instagram.com/reel/CoEBV3eM4QR/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "regular video",
|
||||
"url": "https://www.instagram.com/p/CmCVWoIr9OH/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "reel (isAudioOnly)",
|
||||
"url": "https://www.instagram.com/reel/CoEBV3eM4QR/",
|
||||
"params": {
|
||||
"downloadMode": "audio"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "reel (isAudioMuted)",
|
||||
"url": "https://www.instagram.com/reel/CoEBV3eM4QR/",
|
||||
"params": {
|
||||
"downloadMode": "mute"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "inexistent reel",
|
||||
"url": "https://www.instagram.com/reel/XXXXXXXXXX/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 400,
|
||||
"status": "error"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "inexistent post",
|
||||
"url": "https://www.instagram.com/p/XXXXXXXXXX/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 400,
|
||||
"status": "error"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "post info in an array (for whatever reason??)",
|
||||
"url": "https://www.instagram.com/reel/CrVB9tatUDv/?igshid=blaBlABALALbLABULLSHIT==",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "prone to get rate limited",
|
||||
"url": "https://www.instagram.com/reel/CrO-T7Qo6rq/?igshid=fuckYouNoTrackingIdForYou==",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "ddinstagram link",
|
||||
"url": "https://ddinstagram.com/p/CmCVWoIr9OH/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "d.ddinstagram.com link",
|
||||
"url": "https://d.ddinstagram.com/p/CmCVWoIr9OH/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "g.ddinstagram.com link",
|
||||
"url": "https://g.ddinstagram.com/p/CmCVWoIr9OH/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
}
|
||||
]
|
33
api/src/util/tests/loom.json
Normal file
33
api/src/util/tests/loom.json
Normal file
|
@ -0,0 +1,33 @@
|
|||
[
|
||||
{
|
||||
"name": "1080p video",
|
||||
"url": "https://www.loom.com/share/d165fd054a294d8a8587807bcc50c885",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1080p video (muted)",
|
||||
"url": "https://www.loom.com/share/d165fd054a294d8a8587807bcc50c885",
|
||||
"params": {
|
||||
"downloadMode": "mute"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1080p video (audio only)",
|
||||
"url": "https://www.loom.com/share/d165fd054a294d8a8587807bcc50c885",
|
||||
"params": {
|
||||
"downloadMode": "audio"
|
||||
},
|
||||
"expected": {
|
||||
"code": 400,
|
||||
"status": "error"
|
||||
}
|
||||
}
|
||||
]
|
11
api/src/util/tests/ok.json
Normal file
11
api/src/util/tests/ok.json
Normal file
|
@ -0,0 +1,11 @@
|
|||
[
|
||||
{
|
||||
"name": "regular video",
|
||||
"url": "https://ok.ru/video/7204071410346",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
}
|
||||
]
|
87
api/src/util/tests/pinterest.json
Normal file
87
api/src/util/tests/pinterest.json
Normal file
|
@ -0,0 +1,87 @@
|
|||
[
|
||||
{
|
||||
"name": "regular video",
|
||||
"url": "https://www.pinterest.com/pin/70437485604616/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "regular video (isAudioOnly)",
|
||||
"url": "https://www.pinterest.com/pin/70437485604616/",
|
||||
"params": {
|
||||
"downloadMode": "audio"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "regular video (isAudioMuted)",
|
||||
"url": "https://www.pinterest.com/pin/70437485604616/",
|
||||
"params": {
|
||||
"downloadMode": "mute"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "regular video (.ca TLD)",
|
||||
"url": "https://www.pinterest.ca/pin/70437485604616/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "story",
|
||||
"url": "https://www.pinterest.com/pin/gadget-cool-products-amazon-product-technology-kitchen-gadgets--1084663891475263837/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "regular picture",
|
||||
"url": "https://www.pinterest.com/pin/412994228343400946/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "regular picture (.ca TLD)",
|
||||
"url": "https://www.pinterest.ca/pin/412994228343400946/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "regular gif",
|
||||
"url": "https://www.pinterest.com/pin/643170390530326178/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "regular gif (.ca TLD)",
|
||||
"url": "https://www.pinterest.ca/pin/643170390530326178/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
}
|
||||
]
|
60
api/src/util/tests/reddit.json
Normal file
60
api/src/util/tests/reddit.json
Normal file
|
@ -0,0 +1,60 @@
|
|||
[
|
||||
{
|
||||
"name": "video with audio",
|
||||
"url": "https://www.reddit.com/r/TikTokCringe/comments/wup1fg/id_be_escaping_at_the_first_chance_i_got/?utm_source=share&utm_medium=web2x&context=3",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "video with audio (isAudioOnly)",
|
||||
"url": "https://www.reddit.com/r/TikTokCringe/comments/wup1fg/id_be_escaping_at_the_first_chance_i_got/?utm_source=share&utm_medium=web2x&context=3",
|
||||
"params": {
|
||||
"downloadMode": "audio"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "video with audio (isAudioMuted)",
|
||||
"url": "https://www.reddit.com/r/TikTokCringe/comments/wup1fg/id_be_escaping_at_the_first_chance_i_got/?utm_source=share&utm_medium=web2x&context=3",
|
||||
"params": {
|
||||
"downloadMode": "mute"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "video without audio",
|
||||
"url": "https://www.reddit.com/r/catvideos/comments/ftoeo7/luna_doesnt_want_to_be_bothered_while_shes_napping/?utm_source=share&utm_medium=web2x&context=3",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "actual gif, not looping video",
|
||||
"url": "https://www.reddit.com/r/whenthe/comments/109wqy1/god_really_did_some_trolling/?utm_source=share&utm_medium=web2x&context=3",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "different audio link, live render",
|
||||
"url": "https://www.reddit.com/r/TikTokCringe/comments/15hce91/asian_daddy_kink/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
}
|
||||
]
|
100
api/src/util/tests/rutube.json
Normal file
100
api/src/util/tests/rutube.json
Normal file
|
@ -0,0 +1,100 @@
|
|||
[
|
||||
{
|
||||
"name": "regular video",
|
||||
"url": "https://rutube.ru/video/b2f6c27649907c2fde0af411b03825eb/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "vertical video (isAudioMuted)",
|
||||
"url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/",
|
||||
"params": {
|
||||
"downloadMode": "mute"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "russian region lock",
|
||||
"url": "https://rutube.ru/video/b521653b4f71ece57b8ff54e57ca9b82/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 400,
|
||||
"status": "error"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "vertical video",
|
||||
"url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "yappy",
|
||||
"url": "https://rutube.ru/yappy/c8c32bf7aee04412837656ea26c2b25b/",
|
||||
"canFail": true,
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "shorts",
|
||||
"url": "https://rutube.ru/shorts/935c1afafd0e7d52836d671967d53dac/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "vertical video (isAudioOnly)",
|
||||
"url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/",
|
||||
"params": {
|
||||
"downloadMode": "audio"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "vertical video (isAudioMuted)",
|
||||
"url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/",
|
||||
"params": {
|
||||
"downloadMode": "mute"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "private video",
|
||||
"url": "https://rutube.ru/video/private/1161415be0e686214bb2a498165cab3e/?p=_IL1G8RSnKutunnTYwhZ5A",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "region locked video, should fail",
|
||||
"canFail": true,
|
||||
"url": "https://rutube.ru/video/e7ac82708cc22bd068a3bf6a7004d1b1/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 400,
|
||||
"status": "error"
|
||||
}
|
||||
}
|
||||
]
|
29
api/src/util/tests/snapchat.json
Normal file
29
api/src/util/tests/snapchat.json
Normal file
|
@ -0,0 +1,29 @@
|
|||
[
|
||||
{
|
||||
"name": "spotlight",
|
||||
"url": "https://www.snapchat.com/spotlight/W7_EDlXWTBiXAEEniNoMPwAAYdWxucG9pZmNqAY46j0a5AY46j0YbAAAAAQ",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "shortlinked spotlight",
|
||||
"url": "https://t.snapchat.com/4ZsiBLDi",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "story",
|
||||
"url": "https://www.snapchat.com/add/bazerkmakane",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "picker"
|
||||
}
|
||||
}
|
||||
]
|
106
api/src/util/tests/soundcloud.json
Normal file
106
api/src/util/tests/soundcloud.json
Normal file
|
@ -0,0 +1,106 @@
|
|||
[
|
||||
{
|
||||
"name": "public song (best)",
|
||||
"url": "https://soundcloud.com/l2share77/loona-butterfly?utm_source=clipboard&utm_medium=text&utm_campaign=social_sharing",
|
||||
"params": {
|
||||
"audioFormat": "best"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "public song (mp3, isAudioMuted)",
|
||||
"url": "https://soundcloud.com/l2share77/loona-butterfly?utm_source=clipboard&utm_medium=text&utm_campaign=social_sharing",
|
||||
"params": {
|
||||
"downloadMode": "mute",
|
||||
"audioFormat": "mp3"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "private song",
|
||||
"url": "https://soundcloud.com/user-798052861/asdasdasdsdsd/s-9TqZ7edLJ90",
|
||||
"params": {
|
||||
"audioFormat": "mp3"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "private song (wav, isAudioMuted)",
|
||||
"url": "https://soundcloud.com/user-798052861/asdasdasdsdsd/s-9TqZ7edLJ90",
|
||||
"params": {
|
||||
"downloadMode": "mute",
|
||||
"audioFormat": "wav"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "private song (ogg, isAudioMuted, isAudioOnly)",
|
||||
"url": "https://soundcloud.com/user-798052861/asdasdasdsdsd/s-9TqZ7edLJ90",
|
||||
"params": {
|
||||
"downloadMode": "audio",
|
||||
"audioFormat": "ogg"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "on.soundcloud link",
|
||||
"url": "https://on.soundcloud.com/wLZre",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "on.soundcloud link, different stream type",
|
||||
"url": "https://on.soundcloud.com/AG4c",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "no opus audio, fallback to mp3",
|
||||
"url": "https://soundcloud.com/frums/credits",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "go+ song, should fail",
|
||||
"url": "https://soundcloud.com/dualipa/illusion-1",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 400,
|
||||
"status": "error"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "region locked song, should fail",
|
||||
"canFail": true,
|
||||
"url": "https://soundcloud.com/gotye/somebody-2024-feat-kimbra",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 400,
|
||||
"status": "error"
|
||||
}
|
||||
}
|
||||
]
|
51
api/src/util/tests/streamable.json
Normal file
51
api/src/util/tests/streamable.json
Normal file
|
@ -0,0 +1,51 @@
|
|||
[
|
||||
{
|
||||
"name": "regular video",
|
||||
"url": "https://streamable.com/p9cln4",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "embedded link",
|
||||
"url": "https://streamable.com/e/rsmo56",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "regular video (isAudioOnly)",
|
||||
"url": "https://streamable.com/p9cln4",
|
||||
"params": {
|
||||
"downloadMode": "audio"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "regular video (isAudioMuted)",
|
||||
"url": "https://streamable.com/p9cln4",
|
||||
"params": {
|
||||
"downloadMode": "mute"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "inexistent video",
|
||||
"url": "https://streamable.com/XXXXXX",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 400,
|
||||
"status": "error"
|
||||
}
|
||||
}
|
||||
]
|
47
api/src/util/tests/tiktok.json
Normal file
47
api/src/util/tests/tiktok.json
Normal file
|
@ -0,0 +1,47 @@
|
|||
[
|
||||
{
|
||||
"name": "long link video",
|
||||
"url": "https://www.tiktok.com/@fatfatmillycat/video/7195741644585454894",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "images",
|
||||
"url": "https://www.tiktok.com/@matryoshk4/video/7231234675476532526",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "picker"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "long link inexistent",
|
||||
"url": "https://www.tiktok.com/@blablabla/video/7120851458451417478",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 400,
|
||||
"status": "error"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "short link inexistent",
|
||||
"url": "https://vt.tiktok.com/2p4ewa7/",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 400,
|
||||
"status": "error"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "age restricted video",
|
||||
"url": "https://www.tiktok.com/@.kyle.films/video/7415757181145877793",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 400,
|
||||
"status": "error"
|
||||
}
|
||||
}
|
||||
]
|
49
api/src/util/tests/tumblr.json
Normal file
49
api/src/util/tests/tumblr.json
Normal file
|
@ -0,0 +1,49 @@
|
|||
[
|
||||
{
|
||||
"name": "at.tumblr link",
|
||||
"url": "https://at.tumblr.com/music/704177038274281472/n7x7pr7x4w2b",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "user subdomain link",
|
||||
"url": "https://garfield-69.tumblr.com/post/696499862852780032",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "web app link",
|
||||
"url": "https://www.tumblr.com/rongzhi/707729381162958848/english-added-by-me?source=share",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "tumblr audio",
|
||||
"url": "https://www.tumblr.com/zedneon/737815079301562368/zedneon-ft-mr-sauceman-tech-n9ne-speed-of?source=share",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "tumblr video converted to audio",
|
||||
"url": "https://garfield-69.tumblr.com/post/696499862852780032",
|
||||
"params": {
|
||||
"downloadMode": "audio"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
}
|
||||
]
|
33
api/src/util/tests/twitch.json
Normal file
33
api/src/util/tests/twitch.json
Normal file
|
@ -0,0 +1,33 @@
|
|||
[
|
||||
{
|
||||
"name": "clip",
|
||||
"url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "clip (isAudioOnly)",
|
||||
"url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G",
|
||||
"params": {
|
||||
"downloadMode": "audio"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "clip (isAudioMuted)",
|
||||
"url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G",
|
||||
"params": {
|
||||
"downloadMode": "mute"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
}
|
||||
]
|
213
api/src/util/tests/twitter.json
Normal file
213
api/src/util/tests/twitter.json
Normal file
|
@ -0,0 +1,213 @@
|
|||
[
|
||||
{
|
||||
"name": "regular video",
|
||||
"url": "https://twitter.com/X/status/1697304622749086011",
|
||||
"params": {
|
||||
"audioFormat": "mp3"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "video with mobile web mediaviewer",
|
||||
"url": "https://twitter.com/X/status/1697304622749086011/mediaViewer?currentTweet=1697304622749086011¤tTweetUser=X¤tTweet=1697304622749086011¤tTweetUser=X",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "embedded twitter video",
|
||||
"url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
|
||||
"params": {
|
||||
"audioFormat": "mp3"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "mixed media (image + gif)",
|
||||
"url": "https://twitter.com/sky_mj26/status/1807756010712428565",
|
||||
"params": {
|
||||
"audioFormat": "mp3"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "picker"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "picker: mixed media (3 videos)",
|
||||
"url": "https://twitter.com/DankGameAlert/status/1584726006094794774",
|
||||
"params": {
|
||||
"audioFormat": "mp3"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "picker"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "audio from embedded twitter video (mp3, isAudioOnly)",
|
||||
"url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
|
||||
"params": {
|
||||
"downloadMode": "audio",
|
||||
"audioFormat": "mp3"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "audio from embedded twitter video (best, isAudioOnly)",
|
||||
"url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
|
||||
"params": {
|
||||
"downloadMode": "audio",
|
||||
"audioFormat": "best"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "audio from embedded twitter video (ogg, isAudioOnly, isAudioMuted)",
|
||||
"url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
|
||||
"params": {
|
||||
"downloadMode": "audio",
|
||||
"audioFormat": "best"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "muted embedded twitter video",
|
||||
"url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
|
||||
"params": {
|
||||
"downloadMode": "mute",
|
||||
"audioFormat": "mp3"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "retweeted video",
|
||||
"url": "https://twitter.com/uwukko/status/1696901469633421344",
|
||||
"params": {},
|
||||
"canFail": true,
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "age restricted video",
|
||||
"url": "https://x.com/XSpaces/status/1526955853743546372",
|
||||
"params": {},
|
||||
"canFail": true,
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "twitter voice + x.com link",
|
||||
"url": "https://x.com/eggsaladscreams/status/1693089534886506756?s=46",
|
||||
"params": {},
|
||||
"canFail": true,
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "vxtwitter link",
|
||||
"url": "https://vxtwitter.com/dustbin_nie/status/1624596567188717568?s=20",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "post with 1 image",
|
||||
"url": "https://x.com/PopCrave/status/1815960083475423235",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "post with 4 images",
|
||||
"url": "https://x.com/PopCrave/status/1816260887147114696",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "picker"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "retweeted video, isAudioOnly",
|
||||
"url": "https://twitter.com/hugekiwinuts/status/1618671150829309953?s=46&t=gItGzgwGQQJJaJrO6qc1Pg",
|
||||
"params": {
|
||||
"downloadMode": "mute",
|
||||
"audioFormat": "mp3"
|
||||
},
|
||||
"canFail": true,
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "inexistent post",
|
||||
"url": "https://twitter.com/test/status/9487653",
|
||||
"params": {
|
||||
"audioFormat": "best"
|
||||
},
|
||||
"expected": {
|
||||
"code": 400,
|
||||
"status": "error"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "post with no media content",
|
||||
"url": "https://twitter.com/elonmusk/status/1604617643973124097?s=20",
|
||||
"params": {
|
||||
"audioFormat": "best"
|
||||
},
|
||||
"expected": {
|
||||
"code": 400,
|
||||
"status": "error"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "bookmarked video",
|
||||
"url": "https://twitter.com/i/bookmarks?post_id=1828099210220294314",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "bookmarked photo",
|
||||
"url": "https://twitter.com/i/bookmarks?post_id=1837430141179289876",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
}
|
||||
]
|
64
api/src/util/tests/vimeo.json
Normal file
64
api/src/util/tests/vimeo.json
Normal file
|
@ -0,0 +1,64 @@
|
|||
[
|
||||
{
|
||||
"name": "4k progressive",
|
||||
"url": "https://vimeo.com/288386543",
|
||||
"params": {
|
||||
"videoQuality": "2160"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "720p progressive",
|
||||
"url": "https://vimeo.com/288386543",
|
||||
"params": {
|
||||
"videoQuality": "720"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1080p dash parcel",
|
||||
"url": "https://vimeo.com/967252742",
|
||||
"params": {
|
||||
"videoQuality": "1440"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "720p dash parcel",
|
||||
"url": "https://vimeo.com/967252742",
|
||||
"params": {
|
||||
"videoQuality": "360"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "private video",
|
||||
"url": "https://vimeo.com/903115595/f14d06da38",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "mature video",
|
||||
"url": "https://vimeo.com/973212054",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
}
|
||||
]
|
82
api/src/util/tests/vk.json
Normal file
82
api/src/util/tests/vk.json
Normal file
|
@ -0,0 +1,82 @@
|
|||
[
|
||||
{
|
||||
"name": "clip, defaults",
|
||||
"url": "https://vk.com/clip-57274055_456239788",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "clip, 360",
|
||||
"url": "https://vk.com/clip-57274055_456239788",
|
||||
"params": {
|
||||
"videoQuality": "360"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "clip different link, max",
|
||||
"url": "https://vk.com/clips-57274055?z=clip-57274055_456239788",
|
||||
"params": {
|
||||
"videoQuality": "max"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "video, defaults",
|
||||
"url": "https://vk.com/video-57274055_456239399",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "big 4k video",
|
||||
"url": "https://vk.com/video-1112285_456248465",
|
||||
"params": {
|
||||
"videoQuality": "max"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "short 4k video, 480p, vkvideo.ru domain",
|
||||
"url": "https://vkvideo.ru/video-26006257_456245538",
|
||||
"params": {
|
||||
"videoQuality": "480"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "ancient video (fallback to 240p)",
|
||||
"url": "https://vk.com/video-1959_28496479",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "inexistent video",
|
||||
"url": "https://vk.com/video-53333333_456233333",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 400,
|
||||
"status": "error"
|
||||
}
|
||||
}
|
||||
]
|
58
api/src/util/tests/xiaohongshu.json
Normal file
58
api/src/util/tests/xiaohongshu.json
Normal file
|
@ -0,0 +1,58 @@
|
|||
[
|
||||
{
|
||||
"name": "long link video",
|
||||
"url": "https://www.xiaohongshu.com/discovery/item/6789065900000000210035fc?source=webshare&xhsshare=pc_web&xsec_token=CBustnz_Twf1BSybpe5-D-BzUb-Bx28DPLb418TN9S9Kk&xsec_source=pc_share",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "picker with multiple live photos",
|
||||
"url": "https://www.xiaohongshu.com/explore/67847fa1000000000203e6ed?xsec_token=CBzyP7Y44PPpsM20lgxqrIIJMHqOLemusDsRcmsX0cTpk",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "picker"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "one photo",
|
||||
"url": "https://www.xiaohongshu.com/explore/676e132d000000000b016f68?xsec_token=ABRv6LKzizOFeSaf2HnnBkdBqniB5Ak1fI8tMAHzO31jA",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "short link, might expire eventually",
|
||||
"url": "https://xhslink.com/a/czn4z6c1tic4",
|
||||
"canFail": true,
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "wrong note id",
|
||||
"url": "https://www.xiaohongshu.com/discovery/item/6789065911100000210035fc?source=webshare&xhsshare=pc_web&xsec_token=CBustnz_Twf1BSybpe5-D-BzUb-Bx28DPLb418TN9S9Kk&xsec_source=pc_share",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 400,
|
||||
"status": "error"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "short link, wrong id",
|
||||
"url": "https://xhslink.com/a/aaaaaa",
|
||||
"canFail": true,
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 400,
|
||||
"status": "error"
|
||||
}
|
||||
}
|
||||
]
|
240
api/src/util/tests/youtube.json
Normal file
240
api/src/util/tests/youtube.json
Normal file
|
@ -0,0 +1,240 @@
|
|||
[
|
||||
{
|
||||
"name": "4k video (h264, 1440)",
|
||||
"url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
|
||||
"params": {
|
||||
"youtubeVideoCodec": "h264",
|
||||
"videoQuality": "1440"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "4k video (vp9, 720)",
|
||||
"url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
|
||||
"params": {
|
||||
"youtubeVideoCodec": "vp9",
|
||||
"videoQuality": "720"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "4k video (av1, max)",
|
||||
"url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
|
||||
"params": {
|
||||
"youtubeVideoCodec": "av1",
|
||||
"videoQuality": "max"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "4k video (h264, 720)",
|
||||
"url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
|
||||
"params": {
|
||||
"youtubeVideoCodec": "h264",
|
||||
"videoQuality": "720"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "4k video (vp9, max, isAudioMuted)",
|
||||
"url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
|
||||
"params": {
|
||||
"downloadMode": "mute",
|
||||
"youtubeVideoCodec": "vp9",
|
||||
"videoQuality": "max"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "4k video (h264, max, isAudioMuted)",
|
||||
"url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
|
||||
"params": {
|
||||
"downloadMode": "mute",
|
||||
"youtubeVideoCodec": "h264",
|
||||
"videoQuality": "max"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "4k video (av1, max, isAudioMuted, isAudioOnly, mp3)",
|
||||
"url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
|
||||
"params": {
|
||||
"downloadMode": "audio",
|
||||
"audioFormat": "mp3",
|
||||
"youtubeVideoCodec": "av1",
|
||||
"videoQuality": "max"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "4k video (av1, max, isAudioMuted, isAudioOnly, best)",
|
||||
"url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
|
||||
"params": {
|
||||
"downloadMode": "audio",
|
||||
"audioFormat": "best",
|
||||
"youtubeVideoCodec": "av1",
|
||||
"videoQuality": "max"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "music (mp3, isAudioOnly, isAudioMuted)",
|
||||
"url": "https://music.youtube.com/watch?v=5rGTsvZCEdk&feature=share",
|
||||
"params": {
|
||||
"downloadMode": "audio",
|
||||
"audioFormat": "mp3"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "music (mp3)",
|
||||
"url": "https://music.youtube.com/watch?v=5rGTsvZCEdk&feature=share",
|
||||
"params": {
|
||||
"audioFormat": "mp3"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "audio bitrate higher than video, no vp9 video in response (mp3, isAudioOnly)",
|
||||
"url": "https://www.youtube.com/watch?v=t5nC_ucYBrc",
|
||||
"params": {
|
||||
"downloadMode": "audio",
|
||||
"audioFormat": "mp3"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "short, defaults",
|
||||
"url": "https://www.youtube.com/shorts/r5FpeOJItbw",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "vr 360, av1, max",
|
||||
"url": "https://www.youtube.com/watch?v=hEdzv7D4CbQ",
|
||||
"params": {
|
||||
"youtubeVideoCodec": "vp9",
|
||||
"videoQuality": "max"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "live link, defaults",
|
||||
"url": "https://www.youtube.com/live/ENxZS6PUDuI?feature=shared",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "inexistent video",
|
||||
"url": "https://youtube.com/watch?v=gnjuHYWGEW",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 400,
|
||||
"status": "error"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "broken audioOnly download",
|
||||
"url": "https://www.youtube.com/watch?v=ink80Al5nbw",
|
||||
"params": {
|
||||
"downloadMode": "audio"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "hls video (h264, 1440p)",
|
||||
"url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
|
||||
"params": {
|
||||
"youtubeVideoCodec": "h264",
|
||||
"videoQuality": "1440",
|
||||
"youtubeHLS": true
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "hls video (vp9, 360p)",
|
||||
"url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
|
||||
"params": {
|
||||
"youtubeVideoCodec": "vp9",
|
||||
"videoQuality": "360",
|
||||
"youtubeHLS": true
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "hls video (audio mode)",
|
||||
"url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
|
||||
"params": {
|
||||
"downloadMode": "audio",
|
||||
"youtubeHLS": true
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "hls video (audio mode, best format)",
|
||||
"url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
|
||||
"params": {
|
||||
"downloadMode": "audio",
|
||||
"youtubeHLS": true,
|
||||
"audioFormat": "best"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
}
|
||||
]
|
67
docs/api.md
67
docs/api.md
|
@ -1,9 +1,44 @@
|
|||
# cobalt api documentation
|
||||
this document provides info about methods and acceptable variables for all cobalt api requests.
|
||||
|
||||
> if you are looking for the documentation for the old (7.x) api, you can find
|
||||
> it [here](https://github.com/imputnet/cobalt/blob/7/docs/api.md)
|
||||
<!-- TODO: authorization -->
|
||||
> [!IMPORTANT]
|
||||
> hosted api instances (such as `api.cobalt.tools`) use bot protection and are **not** intended to be used in other projects without explicit permission. if you want to access the cobalt api reliably, you should [host your own instance](/docs/run-an-instance.md) or ask an instance owner for access.
|
||||
|
||||
## authentication
|
||||
an api instance may be configured to require you to authenticate yourself.
|
||||
if this is the case, you will typically receive an [error response](#error-response)
|
||||
with a **`api.auth.<method>.missing`** code, which tells you that a particular method
|
||||
of authentication is required.
|
||||
|
||||
authentication is done by passing the `Authorization` header, containing
|
||||
the authentication scheme and the token:
|
||||
```
|
||||
Authorization: <scheme> <token>
|
||||
```
|
||||
|
||||
currently, cobalt supports two ways of authentication. an instance can
|
||||
choose to configure both, or neither:
|
||||
- [`Api-Key`](#api-key-authentication)
|
||||
- [`Bearer`](#bearer-authentication)
|
||||
|
||||
### api-key authentication
|
||||
the api key authentication is the most straightforward. the instance owner
|
||||
will assign you an api key which you can then use to authenticate like so:
|
||||
```
|
||||
Authorization: Api-Key aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee
|
||||
```
|
||||
|
||||
if you are an instance owner and wish to configure api key authentication,
|
||||
see the [instance](run-an-instance.md#api-key-file-format) documentation!
|
||||
|
||||
### bearer authentication
|
||||
the cobalt server may be configured to issue JWT bearers, which are short-lived
|
||||
tokens intended for use by regular users (e.g. after passing a challenge).
|
||||
currently, cobalt can issue tokens for successfully solved [turnstile](run-an-instance.md#list-of-all-environment-variables)
|
||||
challenge, if the instance has turnstile configured. the resulting token is passed like so:
|
||||
```
|
||||
Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...
|
||||
```
|
||||
|
||||
## POST: `/`
|
||||
cobalt's main processing endpoint.
|
||||
|
@ -11,9 +46,10 @@ cobalt's main processing endpoint.
|
|||
request body type: `application/json`
|
||||
response body type: `application/json`
|
||||
|
||||
```
|
||||
⚠️ you must include Accept and Content-Type headers with every `POST /` request.
|
||||
> [!IMPORTANT]
|
||||
> you must include `Accept` and `Content-Type` headers with every `POST /` request.
|
||||
|
||||
```
|
||||
Accept: application/json
|
||||
Content-Type: application/json
|
||||
```
|
||||
|
@ -28,13 +64,13 @@ Content-Type: application/json
|
|||
| `filenameStyle` | `string` | `classic / pretty / basic / nerdy` | `classic` | changes the way files are named. previews can be seen in the web app. |
|
||||
| `downloadMode` | `string` | `auto / audio / mute` | `auto` | `audio` downloads only the audio, `mute` skips the audio track in videos. |
|
||||
| `youtubeVideoCodec` | `string` | `h264 / av1 / vp9` | `h264` | `h264` is recommended for phones. |
|
||||
| `youtubeDubLang` | `string` | `en / ru / cs / ja / ...` | -- | specifies the language of audio to download, when the youtube video is dubbed |
|
||||
| `youtubeDubBrowserLang` | `boolean` | `true / false` | `false` | uses value from the Accept-Language header for `youtubeDubLang`. |
|
||||
| `youtubeDubLang` | `string` | `en / ru / cs / ja / es-US / ...` | -- | specifies the language of audio to download when a youtube video is dubbed. |
|
||||
| `alwaysProxy` | `boolean` | `true / false` | `false` | tunnels all downloads through the processing server, even when not necessary. |
|
||||
| `disableMetadata` | `boolean` | `true / false` | `false` | disables file metadata when set to `true`. |
|
||||
| `tiktokFullAudio` | `boolean` | `true / false` | `false` | enables download of original sound used in a tiktok video. |
|
||||
| `tiktokH265` | `boolean` | `true / false` | `false` | changes whether 1080p h265 videos are preferred or not. |
|
||||
| `tiktokH265` | `boolean` | `true / false` | `false` | allows h265 videos when enabled. applies to tiktok & xiaohongshu. |
|
||||
| `twitterGif` | `boolean` | `true / false` | `true` | changes whether twitter gifs are converted to .gif |
|
||||
| `youtubeHLS` | `boolean` | `true / false` | `false` | specifies whether to use HLS for downloading video or audio from youtube. |
|
||||
|
||||
### response
|
||||
the response will always be a JSON object containing the `status` key, which will be one of:
|
||||
|
@ -108,3 +144,18 @@ response body type: `application/json`
|
|||
| `commit` | `string` | commit hash |
|
||||
| `branch` | `string` | git branch |
|
||||
| `remote` | `string` | git remote |
|
||||
|
||||
## POST: `/session`
|
||||
|
||||
used for generating JWT tokens, if enabled. currently, cobalt only supports
|
||||
generating tokens when a [turnstile](run-an-instance.md#list-of-all-environment-variables) challenge solution
|
||||
is submitted by the client.
|
||||
|
||||
the turnstile challenge response is submitted via the `cf-turnstile-response` header.
|
||||
### response body
|
||||
| key | type | description |
|
||||
|:----------------|:-----------|:-------------------------------------------------------|
|
||||
| `token` | `string` | a `Bearer` token used for later request authentication |
|
||||
| `exp` | `number` | number in seconds indicating the token lifetime |
|
||||
|
||||
on failure, an [error response](#error-response) is returned.
|
||||
|
|
33
docs/configure-for-youtube.md
Normal file
33
docs/configure-for-youtube.md
Normal file
|
@ -0,0 +1,33 @@
|
|||
# how to configure a cobalt instance for youtube
|
||||
if you get various errors when attempting to download videos that are:
|
||||
publicly available, not region locked, and not age-restricted;
|
||||
then your instance's ip address may have bad reputation.
|
||||
|
||||
in this case you have to use disposable google accounts.
|
||||
there's no other known workaround as of time of writing this document.
|
||||
|
||||
> [!CAUTION]
|
||||
> **NEVER** use your personal google account for downloading videos via any means.
|
||||
> you can use any google accounts that you're willing to sacrifice,
|
||||
> but be prepared to have them **permanently suspended**.
|
||||
>
|
||||
> we recommend that you use accounts that don't link back to your personal google account or identity, just in case.
|
||||
>
|
||||
> use incognito mode when signing in.
|
||||
> we also recommend using vpn/proxy services (such as [mullvad](https://mullvad.net/)).
|
||||
|
||||
1. if you haven't done it already, clone the cobalt repo, go to the cloned directory, and run `pnpm install`
|
||||
|
||||
2. run `pnpm -C api token:youtube`
|
||||
|
||||
3. follow instructions, use incognito mode in your browser when signing in.
|
||||
i cannot stress this enough, but again, **DO NOT USE YOUR PERSONAL GOOGLE ACCOUNT**.
|
||||
|
||||
4. once you have the oauth token, add it to `youtube_oauth` in your cookies file.
|
||||
you can see an [example here](/docs/examples/cookies.example.json).
|
||||
you can have several account tokens in this file, if you like.
|
||||
|
||||
5. all done! enjoy freedom.
|
||||
|
||||
### liability
|
||||
you're responsible for any damage done to any of your google accounts or any other damages. you do this by yourself and at your own risk.
|
|
@ -10,5 +10,8 @@
|
|||
],
|
||||
"twitter": [
|
||||
"auth_token=<replace_this>; ct0=<replace_this>"
|
||||
],
|
||||
"youtube_oauth": [
|
||||
"<output from running `pnpm run token:youtube` in `api` folder goes here>"
|
||||
]
|
||||
}
|
||||
|
|
|
@ -1,30 +1,40 @@
|
|||
services:
|
||||
cobalt-api:
|
||||
image: ghcr.io/imputnet/cobalt:10
|
||||
|
||||
init: true
|
||||
read_only: true
|
||||
restart: unless-stopped
|
||||
container_name: cobalt-api
|
||||
|
||||
init: true
|
||||
|
||||
ports:
|
||||
- 9000:9000/tcp
|
||||
# if you're using a reverse proxy, uncomment the next line and remove the one above (9000:9000/tcp):
|
||||
#- 127.0.0.1:9000:9000
|
||||
# if you use a reverse proxy (such as nginx),
|
||||
# uncomment the next line and remove the one above (9000:9000/tcp):
|
||||
# - 127.0.0.1:9000:9000
|
||||
|
||||
environment:
|
||||
# replace https://api.cobalt.tools/ with your instance's target url in same format
|
||||
API_URL: "https://api.cobalt.tools/"
|
||||
# if you want to use cookies when fetching data from services, uncomment the next line and the lines under volume
|
||||
# replace https://api.url.example/ with your instance's url
|
||||
# or else tunneling functionality won't work properly
|
||||
API_URL: "https://api.url.example/"
|
||||
|
||||
# if you want to use cookies for fetching data from services,
|
||||
# uncomment the next line & volumes section
|
||||
# COOKIE_PATH: "/cookies.json"
|
||||
# see docs/run-an-instance.md for more information
|
||||
|
||||
# it's recommended to configure bot protection or api keys if the instance is public,
|
||||
# see /docs/protect-an-instance.md for more info
|
||||
|
||||
# see /docs/run-an-instance.md for more variables that you can use here
|
||||
|
||||
labels:
|
||||
- com.centurylinklabs.watchtower.scope=cobalt
|
||||
|
||||
# if you want to use cookies when fetching data from services, uncomment volumes and next line
|
||||
#volumes:
|
||||
#- ./cookies.json:/cookies.json
|
||||
# uncomment only if you use the COOKIE_PATH variable
|
||||
# volumes:
|
||||
# - ./cookies.json:/cookies.json
|
||||
|
||||
# update the cobalt image automatically with watchtower
|
||||
# watchtower updates the cobalt image automatically
|
||||
watchtower:
|
||||
image: ghcr.io/containrrr/watchtower
|
||||
restart: unless-stopped
|
||||
|
|
BIN
docs/images/protect-an-instance/add.png
Normal file
BIN
docs/images/protect-an-instance/add.png
Normal file
Binary file not shown.
After ![]() (image error) Size: 15 KiB |
BIN
docs/images/protect-an-instance/created.png
Normal file
BIN
docs/images/protect-an-instance/created.png
Normal file
Binary file not shown.
After ![]() (image error) Size: 28 KiB |
BIN
docs/images/protect-an-instance/domain.png
Normal file
BIN
docs/images/protect-an-instance/domain.png
Normal file
Binary file not shown.
After ![]() (image error) Size: 13 KiB |
BIN
docs/images/protect-an-instance/mode.png
Normal file
BIN
docs/images/protect-an-instance/mode.png
Normal file
Binary file not shown.
After ![]() (image error) Size: 27 KiB |
BIN
docs/images/protect-an-instance/name.png
Normal file
BIN
docs/images/protect-an-instance/name.png
Normal file
Binary file not shown.
After ![]() (image error) Size: 14 KiB |
BIN
docs/images/protect-an-instance/sidebar.png
Normal file
BIN
docs/images/protect-an-instance/sidebar.png
Normal file
Binary file not shown.
After ![]() (image error) Size: 6.5 KiB |
Binary file not shown.
Before ![]() (image error) Size: 4.1 KiB |
Binary file not shown.
Before ![]() (image error) Size: 18 KiB |
Binary file not shown.
Before ![]() (image error) Size: 6.7 KiB |
Binary file not shown.
Before ![]() (image error) Size: 18 KiB |
Binary file not shown.
Before ![]() (image error) Size: 17 KiB |
150
docs/protect-an-instance.md
Normal file
150
docs/protect-an-instance.md
Normal file
|
@ -0,0 +1,150 @@
|
|||
# how to protect your cobalt instance
|
||||
if you keep getting a ton of unknown traffic that hurts the performance of your instance, then it might be a good idea to enable bot protection.
|
||||
|
||||
> [!NOTE]
|
||||
> this tutorial will work reliably on the latest official version of cobalt 10.
|
||||
we can't promise full compatibility with anything else.
|
||||
|
||||
## configure cloudflare turnstile
|
||||
turnstile is a free, safe, and privacy-respecting alternative to captcha.
|
||||
cobalt uses it automatically to weed out bots and automated scripts.
|
||||
your instance doesn't have to be proxied by cloudflare to use turnstile.
|
||||
all you need is a free cloudflare account to get started.
|
||||
|
||||
cloudflare dashboard interface might change over time, but basics should stay the same.
|
||||
|
||||
> [!WARNING]
|
||||
> never share the turnstile secret key, always keep it private. if accidentally exposed, rotate it in widget settings.
|
||||
|
||||
1. open [the cloudflare dashboard](https://dash.cloudflare.com/) and log into your account
|
||||
|
||||
2. once logged in, select `Turnstile` in the sidebar
|
||||
<div align="left">
|
||||
<p>
|
||||
<img src="images/protect-an-instance/sidebar.png" width="250" />
|
||||
</p>
|
||||
</div>
|
||||
|
||||
3. press `Add widget`
|
||||
<div align="left">
|
||||
<p>
|
||||
<img src="images/protect-an-instance/add.png" width="550" />
|
||||
</p>
|
||||
</div>
|
||||
|
||||
4. enter the widget name (can be anything, such as "cobalt")
|
||||
<div align="left">
|
||||
<p>
|
||||
<img src="images/protect-an-instance/name.png" width="450" />
|
||||
</p>
|
||||
</div>
|
||||
|
||||
5. add cobalt frontend domains you want the widget to work with, you can change this list later at any time
|
||||
- if you want to use your processing instance with [cobalt.tools](https://cobalt.tools/) frontend, then add `cobalt.tools` to the list
|
||||
<div align="left">
|
||||
<p>
|
||||
<img src="images/protect-an-instance/domain.png" width="450" />
|
||||
</p>
|
||||
</div>
|
||||
|
||||
6. select `invisible` widget mode
|
||||
<div align="left">
|
||||
<p>
|
||||
<img src="images/protect-an-instance/mode.png" width="450" />
|
||||
</p>
|
||||
</div>
|
||||
|
||||
7. press `create`
|
||||
|
||||
8. keep the page with sitekey and secret key open, you'll need them later.
|
||||
if you closed it, no worries!
|
||||
just open the same turnstile page and press "settings" on your freshly made turnstile widget.
|
||||
|
||||
<div align="left">
|
||||
<p>
|
||||
<img src="images/protect-an-instance/created.png" width="450" />
|
||||
</p>
|
||||
</div>
|
||||
|
||||
you've successfully created a turnstile widget!
|
||||
time to add it to your processing instance.
|
||||
|
||||
### enable turnstile on your processing instance
|
||||
this tutorial assumes that you only have `API_URL` in your `environment` variables list.
|
||||
if you have other variables there, just add new ones after existing ones.
|
||||
|
||||
> [!CAUTION]
|
||||
> never use any values from the tutorial, especially `JWT_SECRET`!
|
||||
|
||||
1. open your `docker-compose.yml` config file in any text editor of choice.
|
||||
2. copy the turnstile sitekey & secret key and paste them to their respective variables.
|
||||
`TURNSTILE_SITEKEY` for the sitekey and `TURNSTILE_SECRET` for the secret key:
|
||||
```yml
|
||||
environment:
|
||||
API_URL: "https://your.instance.url.here.local/"
|
||||
TURNSTILE_SITEKEY: "2x00000000000000000000BB" # use your key
|
||||
TURNSTILE_SECRET: "2x0000000000000000000000000000000AA" # use your key
|
||||
```
|
||||
3. generate a `JWT_SECRET`. we recommend using an alphanumeric collection with a length of at least 64 characters.
|
||||
this string will be used as salt for all JWT keys.
|
||||
|
||||
you can generate a random secret with `pnpm -r token:jwt` or use any other that you like.
|
||||
|
||||
```yml
|
||||
environment:
|
||||
API_URL: "https://your.instance.url.here.local/"
|
||||
TURNSTILE_SITEKEY: "2x00000000000000000000BB" # use your key
|
||||
TURNSTILE_SECRET: "2x0000000000000000000000000000000AA" # use your key
|
||||
JWT_SECRET: "bgBmF4efNCKPirD" # create a new secret, NEVER use this one
|
||||
```
|
||||
4. restart the docker container.
|
||||
|
||||
## configure api keys
|
||||
if you want to use your instance outside of web interface, you'll need an api key!
|
||||
|
||||
> [!NOTE]
|
||||
> this tutorial assumes that you'll keep your keys file locally, on the instance server.
|
||||
> if you wish to upload your file to a remote location,
|
||||
> replace the value for `API_KEYS_URL` with a direct url to the file
|
||||
> and skip the second step.
|
||||
|
||||
> [!WARNING]
|
||||
> when storing keys file remotely, make sure that it's not publicly accessible
|
||||
> and that link to it is either authenticated (via query) or impossible to guess.
|
||||
>
|
||||
> if api keys leak, you'll have to update/remove all UUIDs to revoke them.
|
||||
|
||||
1. create a `keys.json` file following [the schema and example here](/docs//run-an-instance.md#api-key-file-format).
|
||||
|
||||
2. expose the `keys.json` to the docker container:
|
||||
```yml
|
||||
volumes:
|
||||
- ./keys.json:/keys.json:ro # ro - read-only
|
||||
```
|
||||
|
||||
3. add a path to the keys file to container environment:
|
||||
```yml
|
||||
environment:
|
||||
# ... other variables here ...
|
||||
API_KEY_URL: "file:///keys.json"
|
||||
```
|
||||
|
||||
4. restart the docker container.
|
||||
|
||||
## limit access to an instance with api keys but no turnstile
|
||||
by default, api keys are additional, meaning that they're not *required*,
|
||||
but work alongside with turnstile or no auth (regular ip hash rate limiting).
|
||||
|
||||
to always require auth (via keys or turnstile, if configured), set `API_AUTH_REQUIRED` to 1:
|
||||
```yml
|
||||
environment:
|
||||
# ... other variables here ...
|
||||
API_AUTH_REQUIRED: 1
|
||||
```
|
||||
|
||||
- if both keys and turnstile are enabled, then nothing will change.
|
||||
- if only keys are configured, then all requests without a valid api key will be refused.
|
||||
|
||||
### why not make keys exclusive by default?
|
||||
keys may be useful for going around rate limiting,
|
||||
while keeping the rest of api rate limited, with no turnstile in place.
|
|
@ -1,4 +1,4 @@
|
|||
# how to host a cobalt instance yourself
|
||||
# how to run a cobalt instance
|
||||
## using docker compose and package from github (recommended)
|
||||
to run the cobalt docker package, you need to have `docker` and `docker-compose` installed and configured.
|
||||
|
||||
|
@ -54,8 +54,7 @@ sudo apt install nscd
|
|||
sudo service nscd start
|
||||
```
|
||||
|
||||
## list of all environment variables
|
||||
### variables for api
|
||||
## list of environment variables for api
|
||||
| variable name | default | example | description |
|
||||
|:----------------------|:----------|:------------------------|:------------|
|
||||
| `API_PORT` | `9000` | `9000` | changes port from which api server is accessible. |
|
||||
|
@ -72,18 +71,27 @@ sudo service nscd start
|
|||
| `RATELIMIT_MAX` | `20` | `30` | max requests per time window. requests above this amount will be blocked for the rate limit window duration. |
|
||||
| `DURATION_LIMIT` | `10800` | `18000` | max allowed video duration in **seconds**. |
|
||||
| `TUNNEL_LIFESPAN` | `90` | `120` | the duration for which tunnel info is stored in ram, **in seconds**. |
|
||||
| `TURNSTILE_SITEKEY` | ➖ | `1x00000000000000000000BB` | [cloudflare turnstile](https://www.cloudflare.com/products/turnstile/) sitekey used by browser clients to request a challenge.\*\* |
|
||||
| `TURNSTILE_SECRET` | ➖ | `1x0000000000000000000000000000000AA` | [cloudflare turnstile](https://www.cloudflare.com/products/turnstile/) secret used by cobalt to verify the client successfully solved the challenge.\*\* |
|
||||
| `JWT_SECRET` | ➖ | ➖ | the secret used for issuing JWT tokens for request authentication. to choose a value, generate a random, secure, long string (ideally >=16 characters).\*\* |
|
||||
| `JWT_EXPIRY` | `120` | `240` | the duration of how long a cobalt-issued JWT token will remain valid, in seconds. |
|
||||
| `API_KEY_URL` | ➖ | `file://keys.json` | the location of the api key database. for loading API keys, cobalt supports HTTP(S) urls, or local files by specifying a local path using the `file://` protocol. see the "api key file format" below for more details. |
|
||||
| `API_AUTH_REQUIRED` | ➖ | `1` | when set to `1`, the user always needs to be authenticated in some way before they can access the API (either via an api key or via turnstile, if enabled). |
|
||||
| `API_REDIS_URL` | ➖ | `redis://localhost:6379` | when set, cobalt uses redis instead of internal memory for the tunnel cache. |
|
||||
| `API_INSTANCE_COUNT` | ➖ | `2` | supported only on Linux and node.js `>=23.1.0`. when configured, cobalt will spawn multiple sub-instances amongst which requests will be balanced. |
|
||||
| `DISABLED_SERVICES` | ➖ | `bilibili,youtube` | comma-separated list which disables certain services from being used. |
|
||||
|
||||
\* the higher the nice value, the lower the priority. [read more here](https://en.wikipedia.org/wiki/Nice_(Unix)).
|
||||
|
||||
\*\* in order to enable turnstile bot protection, all three **`TURNSTILE_SITEKEY`, `TURNSTILE_SECRET` and `JWT_SECRET`** need to be set.
|
||||
|
||||
#### FREEBIND_CIDR
|
||||
setting a `FREEBIND_CIDR` allows cobalt to pick a random IP for every download and use it for all
|
||||
requests it makes for that particular download. to use freebind in cobalt, you need to follow its [setup instructions](https://github.com/imputnet/freebind.js?tab=readme-ov-file#setup) first. if you configure this option while running cobalt
|
||||
in a docker container, you also need to set the `API_LISTEN_ADDRESS` env to `127.0.0.1`, and set
|
||||
`network_mode` for the container to `host`.
|
||||
|
||||
#### api key file format
|
||||
## api key file format
|
||||
the file is a JSON-serialized object with the following structure:
|
||||
```typescript
|
||||
|
||||
|
|
|
@ -1,37 +0,0 @@
|
|||
# self-troubleshooting cobalt
|
||||
```
|
||||
🚧 this page is work-in-progress. expect more guides to be added in the future!
|
||||
```
|
||||
if any issues occur while using cobalt, you can fix many of them yourself. this document aims to provide guides on how to fix most complicated of them.
|
||||
use wiki navigation on right to jump between solutions.
|
||||
|
||||
## how to fix clipboard pasting in older versions of firefox
|
||||
```
|
||||
🎉 firefox finally supports pasting by default starting from version 125.
|
||||
|
||||
👍 you don't need to follow this tutorial if you're on the latest version of firefox.
|
||||
```
|
||||
you can fix this issue by changing a single preference in `about:config`.
|
||||
|
||||
### steps to enable clipboard functionality
|
||||
1. go to `about:config`:
|
||||
![screenshot showing about:config entered into address bar](images/troubleshooting/clipboard/config.png)
|
||||
|
||||
2. if asked, read what firefox has to say and press "accept the risk and continue".
|
||||
⚠ tinkering with other preferences may break your browser. **do not** edit them unless you know what you're doing.
|
||||
|
||||
![screenshot showing about:config security warning that reads: "proceed with caution. changing advanced configuration preferences can impact firefox performance or security." lower there's a pre-checked checkbox that says: "warn me when i attempt to access these preferences". lowest element is a blue button that says "accept the risk and continue"](images/troubleshooting/clipboard/risk.png)
|
||||
|
||||
3. search for `dom.events.asyncClipboard.readText`
|
||||
|
||||
![screenshot showing "dom.events.asyncclipboard.readtext" entered into search on about:config page](images/troubleshooting/clipboard/search.png)
|
||||
|
||||
4. press the toggle button on very right.
|
||||
|
||||
![screenshot showing "dom.events.asyncclipboard.readtext" preference on about:config page with highlighted toggle button on very right](images/troubleshooting/clipboard/toggle.png)
|
||||
|
||||
5. "false" should change to "true".
|
||||
|
||||
![screenshot showing "dom.events.asyncclipboard.readtext" preference on about:config page, this one with "true" text highlighted](images/troubleshooting/clipboard/toggled.png)
|
||||
|
||||
6. go back to cobalt, reload the page, press `paste` button again. this time it works! enjoy simpler downloading experience :)
|
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue