api: implement support for api keys (#803)

.gitignore

@@ -13,6 +13,7 @@ build
 .env.*
 !.env.example
 cookies.json
+keys.json
 
 # docker
 docker-compose.yml

api/package.json

@@ -33,7 +33,7 @@
         "express-rate-limit": "^6.3.0",
         "ffmpeg-static": "^5.1.0",
         "hls-parser": "^0.10.7",
-        "ipaddr.js": "2.1.0",
+        "ipaddr.js": "2.2.0",
         "nanoid": "^4.0.2",
         "node-cache": "^5.1.2",
         "psl": "1.9.0",

api/src/config.js

@@ -43,6 +43,11 @@ const env = {
                         && process.env.TURNSTILE_SECRET
                         && process.env.JWT_SECRET,
 
+    apiKeyURL: process.env.API_KEY_URL && new URL(process.env.API_KEY_URL),
+    authRequired: process.env.API_AUTH_REQUIRED === '1',
+
+    keyReloadInterval: 900,
+
     enabledServices,
 }
 

api/src/core/api.js

@@ -17,6 +17,7 @@ import { verifyTurnstileToken } from "../security/turnstile.js";
 import { friendlyServiceName } from "../processing/service-alias.js";
 import { verifyStream, getInternalStream } from "../stream/manage.js";
 import { createResponse, normalizeRequest, getIP } from "../processing/request.js";
+import * as APIKeys from "../security/api-keys.js";
 
 const git = {
     branch: await getBranch(),
@@ -57,34 +58,40 @@ export const runAPI = (express, app, __dirname) => {
         git,
     })
 
-    const apiLimiter = rateLimit({
-        windowMs: env.rateLimitWindow * 1000,
-        max: env.rateLimitMax,
-        standardHeaders: true,
-        legacyHeaders: false,
-        keyGenerator: req => {
-            if (req.authorized) {
-                return generateHmac(req.header("Authorization"), ipSalt);
+    const handleRateExceeded = (_, res) => {
+        const { status, body } = createResponse("error", {
+            code: "error.api.rate_exceeded",
+            context: {
+                limit: env.rateLimitWindow
             }
-            return generateHmac(getIP(req), ipSalt);
-        },
-        handler: (req, res) => {
-            const { status, body } = createResponse("error", {
-                code: "error.api.rate_exceeded",
-                context: {
-                    limit: env.rateLimitWindow
-                }
-            });
-            return res.status(status).json(body);
-        }
-    })
+        });
+        return res.status(status).json(body);
+    };
 
-    const apiLimiterStream = rateLimit({
-        windowMs: env.rateLimitWindow * 1000,
-        max: env.rateLimitMax,
+    const sessionLimiter = rateLimit({
+        windowMs: 60000,
+        max: 10,
         standardHeaders: true,
         legacyHeaders: false,
         keyGenerator: req => generateHmac(getIP(req), ipSalt),
+        handler: handleRateExceeded
+    });
+
+    const apiLimiter = rateLimit({
+        windowMs: env.rateLimitWindow * 1000,
+        max: (req) => req.rateLimitMax || env.rateLimitMax,
+        standardHeaders: true,
+        legacyHeaders: false,
+        keyGenerator: req => req.rateLimitKey || generateHmac(getIP(req), ipSalt),
+        handler: handleRateExceeded
+    })
+
+    const apiTunnelLimiter = rateLimit({
+        windowMs: env.rateLimitWindow * 1000,
+        max: (req) => req.rateLimitMax || env.rateLimitMax,
+        standardHeaders: true,
+        legacyHeaders: false,
+        keyGenerator: req => req.rateLimitKey || generateHmac(getIP(req), ipSalt),
         handler: (req, res) => {
             return res.sendStatus(429)
         }
@@ -103,9 +110,6 @@ export const runAPI = (express, app, __dirname) => {
         ...corsConfig,
     }));
 
-    app.post('/', apiLimiter);
-    app.use('/tunnel', apiLimiterStream);
-
     app.post('/', (req, res, next) => {
         if (!acceptRegex.test(req.header('Accept'))) {
             return fail(res, "error.api.header.accept");
@@ -117,7 +121,34 @@ export const runAPI = (express, app, __dirname) => {
     });
 
     app.post('/', (req, res, next) => {
-        if (!env.sessionEnabled) {
+        if (!env.apiKeyURL) {
+            return next();
+        }
+
+        const { success, error } = APIKeys.validateAuthorization(req);
+        if (!success) {
+            // We call next() here if either if:
+            // a) we have user sessions enabled, meaning the request
+            //    will still need a Bearer token to not be rejected, or
+            // b) we do not require the user to be authenticated, and
+            //    so they can just make the request with the regular
+            //    rate limit configuration;
+            // otherwise, we reject the request.
+            if (
+                (env.sessionEnabled || !env.authRequired)
+                && ['missing', 'not_api_key'].includes(error)
+            ) {
+                return next();
+            }
+
+            return fail(res, `error.api.auth.key.${error}`);
+        }
+
+        return next();
+    });
+
+    app.post('/', (req, res, next) => {
+        if (!env.sessionEnabled || req.rateLimitKey) {
             return next();
         }
 
@@ -139,14 +170,16 @@ export const runAPI = (express, app, __dirname) => {
                 return fail(res, "error.api.auth.jwt.invalid");
             }
 
-            req.authorized = true;
+            req.rateLimitKey = generateHmac(req.header("Authorization"), ipSalt);
         } catch {
             return fail(res, "error.api.generic");
         }
         next();
     });
 
+    app.post('/', apiLimiter);
     app.use('/', express.json({ limit: 1024 }));
+
     app.use('/', (err, _, res, next) => {
         if (err) {
             const { status, body } = createResponse("error", {
@@ -158,7 +191,7 @@ export const runAPI = (express, app, __dirname) => {
         next();
     });
 
-    app.post("/session", async (req, res) => {
+    app.post("/session", sessionLimiter, async (req, res) => {
         if (!env.sessionEnabled) {
             return fail(res, "error.api.auth.not_configured")
         }
@@ -228,7 +261,7 @@ export const runAPI = (express, app, __dirname) => {
         }
     })
 
-    app.get('/tunnel', (req, res) => {
+    app.get('/tunnel', apiTunnelLimiter, (req, res) => {
         const id = String(req.query.id);
         const exp = String(req.query.exp);
         const sig = String(req.query.sig);
@@ -310,6 +343,10 @@ export const runAPI = (express, app, __dirname) => {
         setGlobalDispatcher(new ProxyAgent(env.externalProxy))
     }
 
+    if (env.apiKeyURL) {
+        APIKeys.setup(env.apiKeyURL);
+    }
+
     app.listen(env.apiPort, env.listenAddress, () => {
         console.log(`\n` +
             Bright(Cyan("cobalt ")) + Bright("API ^ω⁠^") + "\n" +

api/src/misc/console-text.js

@@ -5,12 +5,19 @@ function t(color, tt) {
 export function Bright(tt) {
     return t("\x1b[1m", tt)
 }
+
 export function Red(tt) {
     return t("\x1b[31m", tt)
 }
+
 export function Green(tt) {
     return t("\x1b[32m", tt)
 }
+
 export function Cyan(tt) {
     return t("\x1b[36m", tt)
 }
+
+export function Yellow(tt) {
+    return t("\x1b[93m", tt)
+}

api/src/security/api-keys.js

@@ -0,0 +1,205 @@
+import { env } from "../config.js";
+import { readFile } from "node:fs/promises";
+import { Yellow } from "../misc/console-text.js";
+import ip from "ipaddr.js";
+
+// this function is a modified variation of code
+// from https://stackoverflow.com/a/32402438/14855621
+const generateWildcardRegex = rule => {
+    var escapeRegex = (str) => str.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, "\\$1");
+    return new RegExp("^" + rule.split("*").map(escapeRegex).join(".*") + "$");
+}
+
+const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+
+let keys = {};
+
+const ALLOWED_KEYS = new Set(['name', 'ips', 'userAgents', 'limit']);
+
+/* Expected format pseudotype:
+** type KeyFileContents = Record<
+**    UUIDv4String,
+**    {
+**        name?: string,
+**        limit?: number | "unlimited",
+**        ips?: CIDRString[],
+**        userAgents?: string[]
+**    }
+** >;
+*/
+
+const validateKeys = (input) => {
+    if (typeof input !== 'object' || input === null) {
+        throw "input is not an object";
+    }
+
+    if (Object.keys(input).some(x => !UUID_REGEX.test(x))) {
+        throw "key file contains invalid key(s)";
+    }
+
+    Object.values(input).forEach(details => {
+        if (typeof details !== 'object' || details === null) {
+            throw "some key(s) are incorrectly configured";
+        }
+
+        const unexpected_key = Object.keys(details).find(k => !ALLOWED_KEYS.has(k));
+        if (unexpected_key) {
+            throw "detail object contains unexpected key: " + unexpected_key;
+        }
+
+        if (details.limit && details.limit !== 'unlimited') {
+            if (typeof details.limit !== 'number')
+                throw "detail object contains invalid limit (not a number)";
+            else if (details.limit < 1)
+                throw "detail object contains invalid limit (not a positive number)";
+        }
+
+        if (details.ips) {
+            if (!Array.isArray(details.ips))
+                throw "details object contains value for `ips` which is not an array";
+
+            const invalid_ip = details.ips.find(
+                addr => typeof addr !== 'string' || (!ip.isValidCIDR(addr) && !ip.isValid(addr))
+            );
+
+            if (invalid_ip) {
+                throw "`ips` in details contains an invalid IP or CIDR range: " + invalid_ip;
+            }
+        }
+
+        if (details.userAgents) {
+            if (!Array.isArray(details.userAgents))
+                throw "details object contains value for `userAgents` which is not an array";
+
+            const invalid_ua = details.userAgents.find(ua => typeof ua !== 'string');
+            if (invalid_ua) {
+                throw "`userAgents` in details contains an invalid user agent: " + invalid_ua;
+            }
+        }
+    });
+}
+
+const formatKeys = (keyData) => {
+    const formatted = {};
+
+    for (let key in keyData) {
+        const data = keyData[key];
+        key = key.toLowerCase();
+
+        formatted[key] = {};
+
+        if (data.limit) {
+            if (data.limit === "unlimited") {
+                data.limit = Infinity;
+            }
+
+            formatted[key].limit = data.limit;
+        }
+
+        if (data.ips) {
+            formatted[key].ips = data.ips.map(addr => {
+                if (ip.isValid(addr)) {
+                    return [ ip.parse(addr), 32 ];
+                }
+
+                return ip.parseCIDR(addr);
+            });
+        }
+
+        if (data.userAgents) {
+            formatted[key].userAgents = data.userAgents.map(generateWildcardRegex);
+        }
+    }
+
+    return formatted;
+}
+
+const loadKeys = async (source) => {
+    let updated;
+    if (source.protocol === 'file:') {
+        const pathname = source.pathname === '/' ? '' : source.pathname;
+        updated = JSON.parse(
+            await readFile(
+                decodeURIComponent(source.host + pathname),
+                'utf8'
+            )
+        );
+    } else {
+        updated = await fetch(source).then(a => a.json());
+    }
+
+    validateKeys(updated);
+    keys = formatKeys(updated);
+}
+
+const wrapLoad = (url) => {
+    loadKeys(url)
+    .then(() => {})
+    .catch((e) => {
+        console.error(`${Yellow('[!]')} Failed loading API keys at ${new Date().toISOString()}.`);
+        console.error('Error:', e);
+    })
+}
+
+const err = (reason) => ({ success: false, error: reason });
+
+export const validateAuthorization = (req) => {
+    const authHeader = req.get('Authorization');
+
+    if (typeof authHeader !== 'string') {
+        return err("missing");
+    }
+
+    const [ authType, keyString ] = authHeader.split(' ', 2);
+    if (authType.toLowerCase() !== 'api-key') {
+        return err("not_api_key");
+    }
+
+    if (!UUID_REGEX.test(keyString) || `${authType} ${keyString}` !== authHeader) {
+        return err("invalid");
+    }
+
+    const matchingKey = keys[keyString.toLowerCase()];
+    if (!matchingKey) {
+        return err("not_found");
+    }
+
+    if (matchingKey.ips) {
+        let addr;
+        try {
+            addr = ip.parse(req.ip);
+        } catch {
+            return err("invalid_ip");
+        }
+
+        const ip_allowed = matchingKey.ips.some(
+            ([ allowed, size ]) => {
+                return addr.kind() === allowed.kind()
+                        && addr.match(allowed, size);
+            }
+        );
+
+        if (!ip_allowed) {
+            return err("ip_not_allowed");
+        }
+    }
+
+    if (matchingKey.userAgents) {
+        const userAgent = req.get('User-Agent');
+        if (!matchingKey.userAgents.some(regex => regex.test(userAgent))) {
+            return err("ua_not_allowed");
+        }
+    }
+
+    req.rateLimitKey = keyString.toLowerCase();
+    req.rateLimitMax = matchingKey.limit;
+
+    return { success: true };
+}
+
+export const setup = (url) => {
+    wrapLoad(url);
+    if (env.keyReloadInterval > 0) {
+        setInterval(() => wrapLoad(url), env.keyReloadInterval * 1000);
+    }
+}

docs/run-an-instance.md

@@ -72,6 +72,8 @@ sudo service nscd start
 | `RATELIMIT_MAX`       | `20`      | `30`                    | max requests per time window. requests above this amount will be blocked for the rate limit window duration. |
 | `DURATION_LIMIT`      | `10800`   | `18000`                 | max allowed video duration in **seconds**. |
 | `TUNNEL_LIFESPAN`     | `90`      | `120`                   | the duration for which tunnel info is stored in ram, **in seconds**. |
+| `API_KEY_URL`         | ➖        | `file://keys.json`      | the location of the api key database. for loading API keys, cobalt supports HTTP(S) urls, or local files by specifying a local path using the `file://` protocol. see the "api key file format" below for more details.  |
+| `API_AUTH_REQUIRED`   | ➖        | `1`                     | when set to `1`, the user always needs to be authenticated in some way before they can access the API (either via an api key or via turnstile, if enabled). |
 
 \* the higher the nice value, the lower the priority. [read more here](https://en.wikipedia.org/wiki/Nice_(Unix)).
 
@@ -80,3 +82,55 @@ setting a `FREEBIND_CIDR` allows cobalt to pick a random IP for every download a
 requests it makes for that particular download. to use freebind in cobalt, you need to follow its [setup instructions](https://github.com/imputnet/freebind.js?tab=readme-ov-file#setup) first. if you configure this option while running cobalt
 in a docker container, you also need to set the `API_LISTEN_ADDRESS` env to `127.0.0.1`, and set
 `network_mode` for the container to `host`.
+
+#### api key file format
+the file is a JSON-serialized object with the following structure:
+```typescript
+
+type KeyFileContents = Record<
+    UUIDv4String,
+    {
+        name?: string,
+        limit?: number | "unlimited",
+        ips?: (CIDRString | IPString)[],
+        userAgents?: string[]
+    }
+>;
+```
+
+where *`UUIDv4String`* is a stringified version of a UUIDv4 identifier.
+- **name** is a field for your own reference, it is not used by cobalt anywhere.
+
+- **`limit`** specifies how many requests the API key can make during the window specified in the `RATELIMIT_WINDOW` env.
+    - when omitted, the limit specified in `RATELIMIT_MAX` will be used.
+    - it can be also set to `"unlimited"`, in which case the API key bypasses all rate limits.
+
+- **`ips`** contains an array of allowlisted IP ranges, which can be specified both as individual ips or CIDR ranges (e.g. *`["192.168.42.69", "2001:db8::48", "10.0.0.0/8", "fe80::/10"]`*).
+    - when specified, only requests from these ip ranges can use the specified api key.
+    - when omitted, any IP can be used to make requests with that API key.
+
+- **`userAgents`** contains an array of allowed user agents, with support for wildcards (e.g. *`["cobaltbot/1.0", "Mozilla/5.0 * Chrome/*"]`*).
+    - when specified, requests with a `user-agent` that does not appear in this array will be rejected.
+    - when omitted, any user agent can be specified to make requests with that API key.
+
+- if both `ips` and `userAgents` are set, the tokens will be limited by both parameters.
+- if cobalt detects any problem with your key file, it will be ignored and a warning will be printed to the console.
+
+an example key file could look like this:
+```json
+{
+    "b5c7160a-b655-4c7a-b500-de839f094550": {
+        "limit": 10,
+        "ips": ["10.0.0.0/8", "192.168.42.42"],
+        "userAgents": ["*Chrome*"]
+    },
+    "b00b1234-a3e5-99b1-c6d1-dba4512ae190": {
+        "limit": "unlimited",
+        "ips": ["192.168.1.2"],
+        "userAgents": ["cobaltbot/1.0"]
+    }
+}
+```
+
+if you are configuring a key file, **do not use the UUID from the example** but instead generate your own. you can do this by running the following command if you have node.js installed:
+`node -e "console.log(crypto.randomUUID())"`

pnpm-lock.yaml

@@ -38,8 +38,8 @@ importers:
         specifier: ^0.10.7
         version: 0.10.9
       ipaddr.js:
-        specifier: 2.1.0
-        version: 2.1.0
+        specifier: 2.2.0
+        version: 2.2.0
       nanoid:
         specifier: ^4.0.2
         version: 4.0.2
@@ -1448,6 +1448,10 @@ packages:
     resolution: {integrity: sha512-LlbxQ7xKzfBusov6UMi4MFpEg0m+mAm9xyNGEduwXMEDuf4WfzB/RZwMVYEd7IKGvh4IUkEXYxtAVu9T3OelJQ==}
     engines: {node: '>= 10'}
 
+  ipaddr.js@2.2.0:
+    resolution: {integrity: sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==}
+    engines: {node: '>= 10'}
+
   is-binary-path@2.1.0:
     resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==}
     engines: {node: '>=8'}
@@ -3471,7 +3475,10 @@ snapshots:
 
   ipaddr.js@1.9.1: {}
 
-  ipaddr.js@2.1.0: {}
+  ipaddr.js@2.1.0:
+    optional: true
+
+  ipaddr.js@2.2.0: {}
 
   is-binary-path@2.1.0:
     dependencies:

web/README.md

@@ -15,7 +15,6 @@ them, you must specify them when building the frontend (or running a vite server
 | `WEB_HOST`           | `cobalt.tools`              | domain on which the frontend will be running. used for meta tags and configuring plausible.              |
 | `WEB_PLAUSIBLE_HOST` | `plausible.io`*             | enables plausible analytics with provided hostname as receiver backend.                                  |
 | `WEB_DEFAULT_API`    | `https://api.cobalt.tools/` | changes url which is used for api requests by frontend clients.                                          |
-| `WEB_TURNSTILE_KEY`  | `1x00000000000000000000AA`  | [cloudflare turnstile](https://www.cloudflare.com/products/turnstile/) public key for antibot protection |
 
 \* don't use plausible.io as receiver backend unless you paid for their cloud service.
    use your own domain when hosting community edition of plausible. refer to their [docs](https://plausible.io/docs) when needed.