From 0fe2fb40e1e1c7dec908c241aff64f40285897d2 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sun, 24 Dec 2023 18:02:56 +0100 Subject: [PATCH 01/17] Check browser compile compatibility in integration tests --- package-lock.json | 1031 ++++++++++++++++++++++++++++++++++++ package.json | 3 +- tests/browser/.gitignore | 1 + tests/browser/index.html | 11 + tests/browser/index.ts | 8 + tests/browser/package.json | 3 + tests/browser/startup.mjs | 31 ++ 7 files changed, 1087 insertions(+), 1 deletion(-) create mode 100644 tests/browser/.gitignore create mode 100644 tests/browser/index.html create mode 100644 tests/browser/index.ts create mode 100644 tests/browser/package.json create mode 100644 tests/browser/startup.mjs diff --git a/package-lock.json b/package-lock.json index 92f5028..6a4523d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,6 +36,7 @@ "eslint-plugin-prettier": "^4.2.1", "node-fetch": "^2.7.0", "prettier": "^2.6.2", + "puppeteer": "^21.6.1", "spinnies": "^0.5.1", "ts-node": "^10.9.1", "tsc-alias": "^1.6.7", @@ -67,6 +68,184 @@ "node": ">=6.0.0" } }, + "node_modules/@babel/code-frame": { + "version": "7.23.5", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.23.5.tgz", + "integrity": "sha512-CgH3s1a96LipHCmSUmYFPwY7MNx8C3avkq7i4Wl3cfa662ldtUe4VM1TPXX70pfmrlWTb6jLqTYrZyT2ZTJBgA==", + "dev": true, + "dependencies": { + "@babel/highlight": "^7.23.4", + "chalk": "^2.4.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/code-frame/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/code-frame/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/code-frame/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/@babel/code-frame/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true + }, + "node_modules/@babel/code-frame/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/@babel/code-frame/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/code-frame/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.22.20", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz", + "integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/highlight": { + "version": "7.23.4", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.23.4.tgz", + "integrity": "sha512-acGdbYSfp2WheJoJm/EBBBLh/ID8KDc64ISZ9DYtBmC8/Q204PZJLHyzeB5qMzJ5trcOkybd78M4x2KWsUq++A==", + "dev": true, + "dependencies": { + "@babel/helper-validator-identifier": "^7.22.20", + "chalk": "^2.4.2", + "js-tokens": "^4.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/highlight/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/@babel/highlight/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true + }, + "node_modules/@babel/highlight/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/@babel/highlight/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/@babel/parser": { "version": "7.23.5", "dev": true, @@ -366,6 +545,27 @@ "node": ">= 8" } }, + "node_modules/@puppeteer/browsers": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-1.9.0.tgz", + "integrity": "sha512-QwguOLy44YBGC8vuPP2nmpX4MUN2FzWbsnvZJtiCzecU3lHmVZkaC1tq6rToi9a200m8RzlVtDyxCS0UIDrxUg==", + "dev": true, + "dependencies": { + "debug": "4.3.4", + "extract-zip": "2.0.1", + "progress": "2.0.3", + "proxy-agent": "6.3.1", + "tar-fs": "3.0.4", + "unbzip2-stream": "1.4.3", + "yargs": "17.7.2" + }, + "bin": { + "browsers": "lib/cjs/main-cli.js" + }, + "engines": { + "node": ">=16.3.0" + } + }, "node_modules/@rollup/pluginutils": { "version": "5.1.0", "dev": true, @@ -442,6 +642,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", + "dev": true + }, "node_modules/@tsconfig/node10": { "version": "1.0.9", "dev": true, @@ -549,6 +755,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "dev": true, + "optional": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "5.62.0", "dev": true, @@ -969,6 +1185,18 @@ "node": ">=0.4.0" } }, + "node_modules/agent-base": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz", + "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==", + "dev": true, + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/ajv": { "version": "6.12.6", "dev": true, @@ -1154,6 +1382,24 @@ "node": "*" } }, + "node_modules/ast-types": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "dev": true, + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/ast-types/node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", + "dev": true + }, "node_modules/asynckit": { "version": "0.4.0", "license": "MIT" @@ -1169,11 +1415,46 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/b4a": { + "version": "1.6.4", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.4.tgz", + "integrity": "sha512-fpWrvyVHEKyeEvbKZTVOeZF3VSKKWtJxFIxX/jaVPf+cLbGUSitjb49pHLqPV2BUNNZ0LcoeEGfE/YCpyDYHIw==", + "dev": true + }, "node_modules/balanced-match": { "version": "1.0.2", "dev": true, "license": "MIT" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/basic-ftp": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.4.tgz", + "integrity": "sha512-8PzkB0arJFV4jJWSGOYR+OEic6aeKMu/osRhBULN6RY0ykby6LKhbmuQ5ublvaas5BOwboah5D87nrHyuh8PPA==", + "dev": true, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/binary-extensions": { "version": "2.2.0", "dev": true, @@ -1206,6 +1487,39 @@ "node": ">=8" } }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "dev": true, + "engines": { + "node": "*" + } + }, "node_modules/cac": { "version": "6.7.14", "dev": true, @@ -1349,6 +1663,19 @@ "node": ">= 6" } }, + "node_modules/chromium-bidi": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-0.5.1.tgz", + "integrity": "sha512-dcCqOgq9fHKExc2R4JZs/oKbOghWpUNFAJODS8WKRtLhp3avtIH5UDCBrutdqZdh3pARogH8y1ObXm87emwb3g==", + "dev": true, + "dependencies": { + "mitt": "3.0.1", + "urlpattern-polyfill": "9.0.0" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, "node_modules/cli-cursor": { "version": "3.1.0", "dev": true, @@ -1360,6 +1687,20 @@ "node": ">=8" } }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/color-convert": { "version": "2.0.1", "dev": true, @@ -1422,11 +1763,46 @@ "dev": true, "license": "MIT" }, + "node_modules/cosmiconfig": { + "version": "8.3.6", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-8.3.6.tgz", + "integrity": "sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==", + "dev": true, + "dependencies": { + "import-fresh": "^3.3.0", + "js-yaml": "^4.1.0", + "parse-json": "^5.2.0", + "path-type": "^4.0.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/d-fischer" + }, + "peerDependencies": { + "typescript": ">=4.9.5" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, "node_modules/create-require": { "version": "1.1.1", "dev": true, "license": "MIT" }, + "node_modules/cross-fetch": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-4.0.0.tgz", + "integrity": "sha512-e4a5N8lVvuLgAWgnCrLr2PP0YyDOTHa9H/Rj54dirp61qXnNq46m82bRhNqIA5VccJtWBvPTFRV3TtvHUKPB1g==", + "dev": true, + "dependencies": { + "node-fetch": "^2.6.12" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "dev": true, @@ -1469,6 +1845,15 @@ "url": "https://github.com/sponsors/fb55" } }, + "node_modules/data-uri-to-buffer": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.1.tgz", + "integrity": "sha512-MZd3VlchQkp8rdend6vrx7MmVDJzSNTBvghvKjirLkD+WTChA3KUf0jkE68Q4UyctNqI11zZO9/x2Yx+ub5Cvg==", + "dev": true, + "engines": { + "node": ">= 14" + } + }, "node_modules/de-indent": { "version": "1.0.2", "dev": true, @@ -1535,6 +1920,20 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/degenerator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "dev": true, + "dependencies": { + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "license": "MIT", @@ -1542,6 +1941,12 @@ "node": ">=0.4.0" } }, + "node_modules/devtools-protocol": { + "version": "0.0.1203626", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1203626.tgz", + "integrity": "sha512-nEzHZteIUZfGCZtTiS1fRpC8UZmsfD1SiyPvaUNvS13dvKf666OAm8YTi0+Ca3n1nLEyu49Cy4+dPWpaHFJk9g==", + "dev": true + }, "node_modules/diff": { "version": "4.0.2", "dev": true, @@ -1638,6 +2043,21 @@ "url": "https://github.com/motdotla/dotenv?sponsor=1" } }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dev": true, + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/enhanced-resolve": { "version": "5.15.0", "dev": true, @@ -1672,6 +2092,15 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dev": true, + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, "node_modules/es-abstract": { "version": "1.22.3", "dev": true, @@ -1797,6 +2226,15 @@ "@esbuild/win32-x64": "0.18.20" } }, + "node_modules/escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/escape-string-regexp": { "version": "4.0.0", "dev": true, @@ -1808,6 +2246,36 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "dev": true, + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/escodegen/node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "engines": { + "node": ">=4.0" + } + }, "node_modules/eslint": { "version": "8.55.0", "dev": true, @@ -2126,6 +2594,19 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true, + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/esquery": { "version": "1.5.0", "dev": true, @@ -2185,6 +2666,26 @@ "node": ">=0.10.0" } }, + "node_modules/extract-zip": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "dev": true, + "dependencies": { + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" + }, + "bin": { + "extract-zip": "cli.js" + }, + "engines": { + "node": ">= 10.17.0" + }, + "optionalDependencies": { + "@types/yauzl": "^2.9.1" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "dev": true, @@ -2195,6 +2696,12 @@ "dev": true, "license": "Apache-2.0" }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "dev": true + }, "node_modules/fast-glob": { "version": "3.3.2", "dev": true, @@ -2239,6 +2746,15 @@ "reusify": "^1.0.4" } }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "dev": true, + "dependencies": { + "pend": "~1.2.0" + } + }, "node_modules/file-entry-cache": { "version": "6.0.1", "dev": true, @@ -2365,6 +2881,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, "node_modules/get-func-name": { "version": "2.0.2", "dev": true, @@ -2387,6 +2912,21 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "dev": true, + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/get-symbol-description": { "version": "1.0.0", "dev": true, @@ -2413,6 +2953,35 @@ "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" } }, + "node_modules/get-uri": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.2.tgz", + "integrity": "sha512-5KLucCJobh8vBY1K07EFV4+cPZH3mrV9YeAruUseCQKHB58SGjjT2l9/eA9LD082IiuMjSlFJEcdJ27TXvbZNw==", + "dev": true, + "dependencies": { + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.0", + "debug": "^4.3.4", + "fs-extra": "^8.1.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/get-uri/node_modules/fs-extra": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", + "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + }, + "engines": { + "node": ">=6 <7 || >=8" + } + }, "node_modules/glob": { "version": "7.2.3", "dev": true, @@ -2615,6 +3184,52 @@ "entities": "^4.4.0" } }, + "node_modules/http-proxy-agent": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.0.tgz", + "integrity": "sha512-+ZT+iBxVUQ1asugqnD6oWoRiS25AkjNfG085dKJGtGxkdwLQrMKU5wJr2bOOFAXzKcTuqq+7fZlTMgG3SRfIYQ==", + "dev": true, + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.2.tgz", + "integrity": "sha512-NmLNjm6ucYwtcUmL7JQC1ZQ57LmHP4lT15FQ8D61nak1rO6DH+fz5qNK2Ap5UN4ZapYICE3/0KodcLYSPsPbaA==", + "dev": true, + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/ignore": { "version": "5.3.0", "dev": true, @@ -2681,6 +3296,12 @@ "node": ">= 0.4" } }, + "node_modules/ip": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.8.tgz", + "integrity": "sha512-PuExPYUiu6qMBQb4l06ecm6T6ujzhmh+MeJcW9wa89PoAz5pvd4zPgN5WJV104mb6S2T1AwNIAaB70JNrLQWhg==", + "dev": true + }, "node_modules/is-array-buffer": { "version": "3.0.2", "dev": true, @@ -2694,6 +3315,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "dev": true + }, "node_modules/is-bigint": { "version": "1.0.4", "dev": true, @@ -2790,6 +3417,15 @@ "node": ">=0.10.0" } }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/is-glob": { "version": "4.0.3", "dev": true, @@ -3011,6 +3647,12 @@ "dev": true, "license": "MIT" }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true + }, "node_modules/js-yaml": { "version": "4.1.0", "dev": true, @@ -3027,6 +3669,12 @@ "dev": true, "license": "MIT" }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "dev": true + }, "node_modules/json-schema-traverse": { "version": "0.4.1", "dev": true, @@ -3087,6 +3735,12 @@ "node": ">= 0.8.0" } }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "dev": true + }, "node_modules/local-pkg": { "version": "0.4.3", "dev": true, @@ -3245,6 +3899,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", + "dev": true + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "dev": true + }, "node_modules/mlly": { "version": "1.4.2", "dev": true, @@ -3304,6 +3970,15 @@ "dev": true, "license": "MIT" }, + "node_modules/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", + "dev": true, + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/node-fetch": { "version": "2.7.0", "dev": true, @@ -3496,6 +4171,39 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/pac-proxy-agent": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.0.1.tgz", + "integrity": "sha512-ASV8yU4LLKBAjqIPMbrgtaKIvxQri/yh2OpI+S6hVa9JRkUI3Y3NPFbfngDtY7oFtSMD3w31Xns89mDa3Feo5A==", + "dev": true, + "dependencies": { + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.0.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.2", + "pac-resolver": "^7.0.0", + "socks-proxy-agent": "^8.0.2" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-resolver": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.0.tgz", + "integrity": "sha512-Fd9lT9vJbHYRACT8OhCbZBbxr6KRSawSovFpy8nDGshaK99S/EBhVIHp9+crhxrsZOuvLpgL1n23iyPg6Rl2hg==", + "dev": true, + "dependencies": { + "degenerator": "^5.0.0", + "ip": "^1.1.8", + "netmask": "^2.0.2" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/parent-module": { "version": "1.0.1", "dev": true, @@ -3507,6 +4215,24 @@ "node": ">=6" } }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/parse5": { "version": "7.1.2", "license": "MIT", @@ -3583,6 +4309,12 @@ "node": "*" } }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", + "dev": true + }, "node_modules/picocolors": { "version": "1.0.0", "dev": true, @@ -3706,6 +4438,59 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "dev": true, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/proxy-agent": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.3.1.tgz", + "integrity": "sha512-Rb5RVBy1iyqOtNl15Cw/llpeLH8bsb37gM1FUfKQ+Wck6xHlbAhWGUFiTRHtkjqGTA5pSHz6+0hrPW/oECihPQ==", + "dev": true, + "dependencies": { + "agent-base": "^7.0.2", + "debug": "^4.3.4", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.2", + "lru-cache": "^7.14.1", + "pac-proxy-agent": "^7.0.1", + "proxy-from-env": "^1.1.0", + "socks-proxy-agent": "^8.0.2" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "dev": true + }, + "node_modules/pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dev": true, + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/punycode": { "version": "2.3.1", "dev": true, @@ -3714,6 +4499,41 @@ "node": ">=6" } }, + "node_modules/puppeteer": { + "version": "21.6.1", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-21.6.1.tgz", + "integrity": "sha512-O+pbc61oj8ln6m8EJKncrsQFmytgRyFYERtk190PeLbJn5JKpmmynn2p1PiFrlhCitAQXLJ0MOy7F0TeyCRqBg==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "@puppeteer/browsers": "1.9.0", + "cosmiconfig": "8.3.6", + "puppeteer-core": "21.6.1" + }, + "bin": { + "puppeteer": "lib/esm/puppeteer/node/cli.js" + }, + "engines": { + "node": ">=16.13.2" + } + }, + "node_modules/puppeteer-core": { + "version": "21.6.1", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-21.6.1.tgz", + "integrity": "sha512-0chaaK/RL9S1U3bsyR4fUeUfoj51vNnjWvXgG6DcsyMjwYNpLcAThv187i1rZCo7QhJP0wZN8plQkjNyrq2h+A==", + "dev": true, + "dependencies": { + "@puppeteer/browsers": "1.9.0", + "chromium-bidi": "0.5.1", + "cross-fetch": "4.0.0", + "debug": "4.3.4", + "devtools-protocol": "0.0.1203626", + "ws": "8.15.1" + }, + "engines": { + "node": ">=16.13.2" + } + }, "node_modules/queue-lit": { "version": "1.5.2", "dev": true, @@ -3741,6 +4561,12 @@ ], "license": "MIT" }, + "node_modules/queue-tick": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", + "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==", + "dev": true + }, "node_modules/react-is": { "version": "18.2.0", "dev": true, @@ -3773,6 +4599,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/resolve": { "version": "1.22.8", "dev": true, @@ -3998,6 +4833,50 @@ "node": ">=8" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "dev": true, + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.7.1.tgz", + "integrity": "sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==", + "dev": true, + "dependencies": { + "ip": "^2.0.0", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.13.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.2.tgz", + "integrity": "sha512-8zuqoLv1aP/66PHF5TqwJ7Czm3Yv32urJQHrVyhD7mmA6d61Zv8cIXQYPTWwmg6qlupnPvs/QKDmfa4P/qct2g==", + "dev": true, + "dependencies": { + "agent-base": "^7.0.2", + "debug": "^4.3.4", + "socks": "^2.7.1" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/socks/node_modules/ip": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz", + "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==", + "dev": true + }, "node_modules/source-map": { "version": "0.6.1", "dev": true, @@ -4122,6 +5001,16 @@ "dev": true, "license": "MIT" }, + "node_modules/streamx": { + "version": "2.15.6", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.15.6.tgz", + "integrity": "sha512-q+vQL4AAz+FdfT137VF69Cc/APqUbxy+MDOImRrMvchJpigHj9GksgDU2LYbO9rx7RX6osWgxJB2WxhYv4SZAw==", + "dev": true, + "dependencies": { + "fast-fifo": "^1.1.0", + "queue-tick": "^1.0.1" + } + }, "node_modules/string-argv": { "version": "0.3.2", "dev": true, @@ -4130,6 +5019,20 @@ "node": ">=0.6.19" } }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/string.prototype.trim": { "version": "1.2.8", "dev": true, @@ -4243,6 +5146,28 @@ "node": ">=6" } }, + "node_modules/tar-fs": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.4.tgz", + "integrity": "sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w==", + "dev": true, + "dependencies": { + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + } + }, + "node_modules/tar-stream": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.6.tgz", + "integrity": "sha512-B/UyjYwPpMBv+PaFSWAmtYjwdrlEaZQEhMIBFNC5oEG8lpiW8XjcSdmEaClj28ArfKScKHs2nshz3k2le6crsg==", + "dev": true, + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, "node_modules/test-exclude": { "version": "6.0.0", "dev": true, @@ -4261,6 +5186,12 @@ "dev": true, "license": "MIT" }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "dev": true + }, "node_modules/tinybench": { "version": "2.5.1", "dev": true, @@ -4518,6 +5449,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/unbzip2-stream": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", + "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", + "dev": true, + "dependencies": { + "buffer": "^5.2.1", + "through": "^2.3.8" + } + }, "node_modules/undici-types": { "version": "5.26.5", "dev": true, @@ -4543,6 +5484,12 @@ "punycode": "^2.1.0" } }, + "node_modules/urlpattern-polyfill": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/urlpattern-polyfill/-/urlpattern-polyfill-9.0.0.tgz", + "integrity": "sha512-WHN8KDQblxd32odxeIgo83rdVDE2bvdkb86it7bMhYZwWKJz0+O0RK/eZiHYnM+zgt/U7hAHOlCQGfjjvSkw2g==", + "dev": true + }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "dev": true, @@ -4887,16 +5834,100 @@ "node": ">=8" } }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/wrappy": { "version": "1.0.2", "dev": true, "license": "ISC" }, + "node_modules/ws": { + "version": "8.15.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.15.1.tgz", + "integrity": "sha512-W5OZiCjXEmk0yZ66ZN82beM5Sz7l7coYxpRkzS+p9PP+ToQry8szKh+61eNktr7EA9DOwvFGhfC605jDHbP6QQ==", + "dev": true, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "engines": { + "node": ">=10" + } + }, "node_modules/yallist": { "version": "4.0.0", "dev": true, "license": "ISC" }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "dev": true, + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + }, "node_modules/yn": { "version": "3.1.1", "dev": true, diff --git a/package.json b/package.json index 17915a7..7395565 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "test": "vitest run", "test:dev": "ts-node ./src/dev-cli.ts", "test:watch": "vitest", - "test:integration": "node ./tests/cjs && node ./tests/esm", + "test:integration": "node ./tests/cjs && node ./tests/esm && node ./tests/browser", "test:coverage": "vitest run --coverage", "lint": "eslint --ext .ts,.js src/", "lint:fix": "eslint --fix --ext .ts,.js src/", @@ -65,6 +65,7 @@ "eslint-plugin-prettier": "^4.2.1", "node-fetch": "^2.7.0", "prettier": "^2.6.2", + "puppeteer": "^21.6.1", "spinnies": "^0.5.1", "ts-node": "^10.9.1", "tsc-alias": "^1.6.7", diff --git a/tests/browser/.gitignore b/tests/browser/.gitignore new file mode 100644 index 0000000..1521c8b --- /dev/null +++ b/tests/browser/.gitignore @@ -0,0 +1 @@ +dist diff --git a/tests/browser/index.html b/tests/browser/index.html new file mode 100644 index 0000000..a27e53d --- /dev/null +++ b/tests/browser/index.html @@ -0,0 +1,11 @@ + + + + + + Browser integration test + + + + + diff --git a/tests/browser/index.ts b/tests/browser/index.ts new file mode 100644 index 0000000..7ad11d1 --- /dev/null +++ b/tests/browser/index.ts @@ -0,0 +1,8 @@ +import { makeProviders, makeStandardFetcher, targets } from '../../lib/index.mjs'; + +(window as any).TEST = () => { + makeProviders({ + fetcher: makeStandardFetcher(fetch), + target: targets.ALL, + }); +} diff --git a/tests/browser/package.json b/tests/browser/package.json new file mode 100644 index 0000000..727f329 --- /dev/null +++ b/tests/browser/package.json @@ -0,0 +1,3 @@ +{ + "main": "startup.mjs" +} diff --git a/tests/browser/startup.mjs b/tests/browser/startup.mjs new file mode 100644 index 0000000..fc43bd5 --- /dev/null +++ b/tests/browser/startup.mjs @@ -0,0 +1,31 @@ +import { build, preview } from 'vite'; +import puppeteer from 'puppeteer'; +import { fileURLToPath } from 'url'; +import { dirname } from 'path'; + +const root = dirname(fileURLToPath(import.meta.url)); + +await build({ + root, +}); +const server = await preview({ + root, +}); +let browser; +try { + browser = await puppeteer.launch({ + headless: 'new', + args: ['--no-sandbox', '--disable-setuid-sandbox'], + }); + const page = await browser.newPage(); + await page.goto(server.resolvedUrls.local[0]); + await page.waitForFunction('!!window.TEST', { timeout: 5000 }); + await page.evaluate(() => { + window.TEST(); + }); +} finally { + server.httpServer.close(); + await browser.close(); +} + +console.log('Success!'); From a64a80cf127a480f0b414afdd9665c237018e025 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sun, 24 Dec 2023 19:18:46 +0100 Subject: [PATCH 02/17] Rename flags + rename targets + add disallowed section to feature mapping --- src/__test__/providerTests.ts | 13 ++++ src/__test__/providers/checks.test.ts | 4 +- src/__test__/utils/features.test.ts | 77 +++++++++++++++++++++ src/main/builder.ts | 7 +- src/main/targets.ts | 32 +++++++-- src/providers/embeds/febbox/hls.ts | 2 +- src/providers/embeds/febbox/mp4.ts | 2 +- src/providers/embeds/mp4upload.ts | 2 +- src/providers/embeds/smashystream/dued.ts | 2 +- src/providers/embeds/smashystream/video1.ts | 2 +- src/providers/embeds/streamsb.ts | 2 +- src/providers/embeds/upcloud.ts | 2 +- src/providers/embeds/upstream.ts | 2 +- src/providers/sources/flixhq/index.ts | 2 +- src/providers/sources/gomovies/index.ts | 2 +- src/providers/sources/kissasian/index.ts | 2 +- src/providers/sources/remotestream.ts | 6 +- src/providers/sources/showbox/index.ts | 2 +- src/providers/sources/smashystream/index.ts | 2 +- src/providers/sources/zoechip/index.ts | 2 +- 20 files changed, 143 insertions(+), 24 deletions(-) create mode 100644 src/__test__/utils/features.test.ts diff --git a/src/__test__/providerTests.ts b/src/__test__/providerTests.ts index a8c0fa8..551a3ec 100644 --- a/src/__test__/providerTests.ts +++ b/src/__test__/providerTests.ts @@ -15,40 +15,52 @@ export function makeProviderMocks() { const sourceA = { id: 'a', + name: 'A', rank: 1, disabled: false, + flags: [], } as Sourcerer; const sourceB = { id: 'b', + name: 'B', rank: 2, disabled: false, + flags: [], } as Sourcerer; const sourceCDisabled = { id: 'c', + name: 'C', rank: 3, disabled: true, + flags: [], } as Sourcerer; const sourceAHigherRank = { id: 'a', + name: 'A', rank: 100, disabled: false, + flags: [], } as Sourcerer; const sourceGSameRankAsA = { id: 'g', + name: 'G', rank: 1, disabled: false, + flags: [], } as Sourcerer; const fullSourceYMovie = { id: 'y', name: 'Y', rank: 105, scrapeMovie: vi.fn(), + flags: [], } as Sourcerer; const fullSourceYShow = { id: 'y', name: 'Y', rank: 105, scrapeShow: vi.fn(), + flags: [], } as Sourcerer; const fullSourceZBoth = { id: 'z', @@ -56,6 +68,7 @@ const fullSourceZBoth = { rank: 106, scrapeMovie: vi.fn(), scrapeShow: vi.fn(), + flags: [], } as Sourcerer; const embedD = { diff --git a/src/__test__/providers/checks.test.ts b/src/__test__/providers/checks.test.ts index dca4e73..7ffa845 100644 --- a/src/__test__/providers/checks.test.ts +++ b/src/__test__/providers/checks.test.ts @@ -1,12 +1,14 @@ import { mockEmbeds, mockSources } from '@/__test__/providerTests'; +import { FeatureMap } from '@/main/targets.ts'; import { getProviders } from '@/providers/get'; import { vi, describe, it, expect, afterEach } from 'vitest'; const mocks = await vi.hoisted(async () => (await import('../providerTests.ts')).makeProviderMocks()); vi.mock('@/providers/all', () => mocks); -const features = { +const features: FeatureMap = { requires: [], + disallowed: [] } describe('getProviders()', () => { diff --git a/src/__test__/utils/features.test.ts b/src/__test__/utils/features.test.ts new file mode 100644 index 0000000..0cb6a4e --- /dev/null +++ b/src/__test__/utils/features.test.ts @@ -0,0 +1,77 @@ +import { FeatureMap, Flags, flags, flagsAllowedInFeatures } from "@/main/targets"; +import { describe, it, expect } from "vitest"; + +describe('flagsAllowedInFeatures()', () => { + function checkFeatures(featureMap: FeatureMap, flags: Flags[], output: boolean) { + expect(flagsAllowedInFeatures(featureMap, flags)).toEqual(output); + } + + it('should check required correctly', () => { + checkFeatures({ + requires: [], + disallowed: [] + }, [], true); + checkFeatures({ + requires: [flags.CORS_ALLOWED], + disallowed: [] + }, [flags.CORS_ALLOWED], true); + checkFeatures({ + requires: [flags.CORS_ALLOWED], + disallowed: [] + }, [], false); + checkFeatures({ + requires: [flags.CORS_ALLOWED, flags.IP_LOCKED], + disallowed: [] + }, [flags.CORS_ALLOWED, flags.IP_LOCKED], true); + checkFeatures({ + requires: [flags.IP_LOCKED], + disallowed: [] + }, [flags.CORS_ALLOWED], false); + checkFeatures({ + requires: [flags.IP_LOCKED], + disallowed: [] + }, [], false); + }); + + it('should check disallowed correctly', () => { + checkFeatures({ + requires: [], + disallowed: [] + }, [], true); + checkFeatures({ + requires: [], + disallowed: [flags.CORS_ALLOWED] + }, [], true); + checkFeatures({ + requires: [], + disallowed: [flags.CORS_ALLOWED] + }, [flags.CORS_ALLOWED], false); + checkFeatures({ + requires: [], + disallowed: [flags.CORS_ALLOWED] + }, [flags.IP_LOCKED], true); + checkFeatures({ + requires: [], + disallowed: [flags.CORS_ALLOWED, flags.IP_LOCKED] + }, [flags.CORS_ALLOWED], false); + }); + + it('should pass mixed tests', () => { + checkFeatures({ + requires: [flags.CORS_ALLOWED], + disallowed: [flags.IP_LOCKED] + }, [], false); + checkFeatures({ + requires: [flags.CORS_ALLOWED], + disallowed: [flags.IP_LOCKED] + }, [flags.CORS_ALLOWED], true); + checkFeatures({ + requires: [flags.CORS_ALLOWED], + disallowed: [flags.IP_LOCKED] + }, [flags.IP_LOCKED], false); + checkFeatures({ + requires: [flags.CORS_ALLOWED], + disallowed: [flags.IP_LOCKED] + }, [flags.IP_LOCKED, flags.CORS_ALLOWED], false); + }); +}); diff --git a/src/main/builder.ts b/src/main/builder.ts index 0322dbd..2ad6911 100644 --- a/src/main/builder.ts +++ b/src/main/builder.ts @@ -5,7 +5,7 @@ import { scrapeIndividualEmbed, scrapeInvidualSource } from '@/main/individualRu import { ScrapeMedia } from '@/main/media'; import { MetaOutput, getAllEmbedMetaSorted, getAllSourceMetaSorted, getSpecificId } from '@/main/meta'; import { RunOutput, runAllProviders } from '@/main/runner'; -import { Targets, getTargetFeatures } from '@/main/targets'; +import { Targets, flags, getTargetFeatures } from '@/main/targets'; import { EmbedOutput, SourcererOutput } from '@/providers/base'; import { getProviders } from '@/providers/get'; @@ -19,6 +19,10 @@ export interface ProviderBuilderOptions { // target of where the streams will be used target: Targets; + + // Set this to true, if the requests will have the same IP as + // the device that the stream will be played on + consistentIpForRequests?: boolean; } export interface RunnerOptions { @@ -82,6 +86,7 @@ export interface ProviderControls { export function makeProviders(ops: ProviderBuilderOptions): ProviderControls { const features = getTargetFeatures(ops.target); + if (!ops.consistentIpForRequests) features.disallowed.push(flags.IP_LOCKED); const list = getProviders(features); const providerRunnerOps = { features, diff --git a/src/main/targets.ts b/src/main/targets.ts index eb791c2..66d6040 100644 --- a/src/main/targets.ts +++ b/src/main/targets.ts @@ -1,31 +1,51 @@ export const flags = { - NO_CORS: 'no-cors', + // CORS are set to allow any origin + CORS_ALLOWED: 'cors-allowed', + + // the stream is locked on IP, so only works if + // request maker is same as player (not compatible with proxies) IP_LOCKED: 'ip-locked', } as const; export type Flags = (typeof flags)[keyof typeof flags]; export const targets = { + // browser with CORS restrictions BROWSER: 'browser', + + // browser, but no CORS restrictions through a browser extension + BROWSER_EXTENSION: 'browser-extension', + + // native app, so no restrictions in what can be played NATIVE: 'native', - ALL: 'all', + + // any target, no target restrictions + ANY: 'any', } as const; export type Targets = (typeof targets)[keyof typeof targets]; export type FeatureMap = { - requires: readonly Flags[]; + requires: Flags[]; + disallowed: Flags[]; }; export const targetToFeatures: Record = { browser: { - requires: [flags.NO_CORS], + requires: [flags.CORS_ALLOWED], + disallowed: [], + }, + 'browser-extension': { + requires: [], + disallowed: [], }, native: { requires: [], + disallowed: [], }, - all: { + any: { requires: [], + disallowed: [], }, } as const; @@ -36,5 +56,7 @@ export function getTargetFeatures(target: Targets): FeatureMap { export function flagsAllowedInFeatures(features: FeatureMap, inputFlags: Flags[]): boolean { const hasAllFlags = features.requires.every((v) => inputFlags.includes(v)); if (!hasAllFlags) return false; + const hasDisallowedFlag = features.disallowed.some((v) => inputFlags.includes(v)); + if (hasDisallowedFlag) return false; return true; } diff --git a/src/providers/embeds/febbox/hls.ts b/src/providers/embeds/febbox/hls.ts index 58478ca..e0d48e0 100644 --- a/src/providers/embeds/febbox/hls.ts +++ b/src/providers/embeds/febbox/hls.ts @@ -38,7 +38,7 @@ export const febboxHlsScraper = makeEmbed({ return { stream: { type: 'hls', - flags: [flags.NO_CORS], + flags: [flags.CORS_ALLOWED], captions: await getSubtitles(ctx, id, firstStream.fid, type as MediaTypes, season, episode), playlist: `https://www.febbox.com/hls/main/${firstStream.oss_fid}.m3u8`, }, diff --git a/src/providers/embeds/febbox/mp4.ts b/src/providers/embeds/febbox/mp4.ts index 086f191..520a35e 100644 --- a/src/providers/embeds/febbox/mp4.ts +++ b/src/providers/embeds/febbox/mp4.ts @@ -43,7 +43,7 @@ export const febboxMp4Scraper = makeEmbed({ captions: await getSubtitles(ctx, id, fid, type, episode, season), qualities, type: 'file', - flags: [flags.NO_CORS], + flags: [flags.CORS_ALLOWED], }, }; }, diff --git a/src/providers/embeds/mp4upload.ts b/src/providers/embeds/mp4upload.ts index 81c2f94..72aef8a 100644 --- a/src/providers/embeds/mp4upload.ts +++ b/src/providers/embeds/mp4upload.ts @@ -17,7 +17,7 @@ export const mp4uploadScraper = makeEmbed({ return { stream: { type: 'file', - flags: [flags.NO_CORS], + flags: [flags.CORS_ALLOWED], captions: [], qualities: { '1080': { diff --git a/src/providers/embeds/smashystream/dued.ts b/src/providers/embeds/smashystream/dued.ts index 89c0814..df45af2 100644 --- a/src/providers/embeds/smashystream/dued.ts +++ b/src/providers/embeds/smashystream/dued.ts @@ -60,7 +60,7 @@ export const smashyStreamDScraper = makeEmbed({ stream: { playlist: playlistRes, type: 'hls', - flags: [flags.NO_CORS], + flags: [flags.CORS_ALLOWED], captions: [], }, }; diff --git a/src/providers/embeds/smashystream/video1.ts b/src/providers/embeds/smashystream/video1.ts index 3dde571..61172fc 100644 --- a/src/providers/embeds/smashystream/video1.ts +++ b/src/providers/embeds/smashystream/video1.ts @@ -45,7 +45,7 @@ export const smashyStreamFScraper = makeEmbed({ stream: { playlist: res.sourceUrls[0], type: 'hls', - flags: [flags.NO_CORS], + flags: [flags.CORS_ALLOWED], captions, }, }; diff --git a/src/providers/embeds/streamsb.ts b/src/providers/embeds/streamsb.ts index d15b320..3b0cb7d 100644 --- a/src/providers/embeds/streamsb.ts +++ b/src/providers/embeds/streamsb.ts @@ -157,7 +157,7 @@ export const streamsbScraper = makeEmbed({ return { stream: { type: 'file', - flags: [flags.NO_CORS], + flags: [flags.CORS_ALLOWED], qualities, captions: [], }, diff --git a/src/providers/embeds/upcloud.ts b/src/providers/embeds/upcloud.ts index d31cf72..514d4f1 100644 --- a/src/providers/embeds/upcloud.ts +++ b/src/providers/embeds/upcloud.ts @@ -121,7 +121,7 @@ export const upcloudScraper = makeEmbed({ stream: { type: 'hls', playlist: sources.file, - flags: [flags.NO_CORS], + flags: [flags.CORS_ALLOWED], captions, }, }; diff --git a/src/providers/embeds/upstream.ts b/src/providers/embeds/upstream.ts index 852aac7..2b62ee9 100644 --- a/src/providers/embeds/upstream.ts +++ b/src/providers/embeds/upstream.ts @@ -24,7 +24,7 @@ export const upstreamScraper = makeEmbed({ stream: { type: 'hls', playlist: link[1], - flags: [flags.NO_CORS], + flags: [flags.CORS_ALLOWED], captions: [], }, }; diff --git a/src/providers/sources/flixhq/index.ts b/src/providers/sources/flixhq/index.ts index 90fe251..b866677 100644 --- a/src/providers/sources/flixhq/index.ts +++ b/src/providers/sources/flixhq/index.ts @@ -9,7 +9,7 @@ export const flixhqScraper = makeSourcerer({ id: 'flixhq', name: 'FlixHQ', rank: 100, - flags: [flags.NO_CORS], + flags: [flags.CORS_ALLOWED], async scrapeMovie(ctx) { const id = await getFlixhqId(ctx, ctx.media); if (!id) throw new NotFoundError('no search results match'); diff --git a/src/providers/sources/gomovies/index.ts b/src/providers/sources/gomovies/index.ts index ce4a428..ff43411 100644 --- a/src/providers/sources/gomovies/index.ts +++ b/src/providers/sources/gomovies/index.ts @@ -13,7 +13,7 @@ export const goMoviesScraper = makeSourcerer({ id: 'gomovies', name: 'GOmovies', rank: 110, - flags: [flags.NO_CORS], + flags: [flags.CORS_ALLOWED], async scrapeShow(ctx) { const search = await ctx.proxiedFetcher(`/ajax/search`, { method: 'POST', diff --git a/src/providers/sources/kissasian/index.ts b/src/providers/sources/kissasian/index.ts index 67111ca..d856ea5 100644 --- a/src/providers/sources/kissasian/index.ts +++ b/src/providers/sources/kissasian/index.ts @@ -13,7 +13,7 @@ export const kissAsianScraper = makeSourcerer({ id: 'kissasian', name: 'KissAsian', rank: 130, - flags: [flags.NO_CORS], + flags: [flags.CORS_ALLOWED], disabled: true, async scrapeShow(ctx) { diff --git a/src/providers/sources/remotestream.ts b/src/providers/sources/remotestream.ts index 696d897..f3ea478 100644 --- a/src/providers/sources/remotestream.ts +++ b/src/providers/sources/remotestream.ts @@ -8,7 +8,7 @@ export const remotestreamScraper = makeSourcerer({ id: 'remotestream', name: 'Remote Stream', rank: 55, - flags: [flags.NO_CORS], + flags: [flags.CORS_ALLOWED], async scrapeShow(ctx) { const seasonNumber = ctx.media.season.number; const episodeNumber = ctx.media.episode.number; @@ -26,7 +26,7 @@ export const remotestreamScraper = makeSourcerer({ captions: [], playlist: playlistLink, type: 'hls', - flags: [flags.NO_CORS], + flags: [flags.CORS_ALLOWED], }, }; }, @@ -44,7 +44,7 @@ export const remotestreamScraper = makeSourcerer({ captions: [], playlist: playlistLink, type: 'hls', - flags: [flags.NO_CORS], + flags: [flags.CORS_ALLOWED], }, }; }, diff --git a/src/providers/sources/showbox/index.ts b/src/providers/sources/showbox/index.ts index 94164ee..064011e 100644 --- a/src/providers/sources/showbox/index.ts +++ b/src/providers/sources/showbox/index.ts @@ -47,7 +47,7 @@ export const showboxScraper = makeSourcerer({ id: 'showbox', name: 'Showbox', rank: 300, - flags: [flags.NO_CORS], + flags: [flags.CORS_ALLOWED], scrapeShow: comboScraper, scrapeMovie: comboScraper, }); diff --git a/src/providers/sources/smashystream/index.ts b/src/providers/sources/smashystream/index.ts index c5a8b27..c7601f8 100644 --- a/src/providers/sources/smashystream/index.ts +++ b/src/providers/sources/smashystream/index.ts @@ -58,7 +58,7 @@ export const smashyStreamScraper = makeSourcerer({ id: 'smashystream', name: 'SmashyStream', rank: 70, - flags: [flags.NO_CORS], + flags: [flags.CORS_ALLOWED], scrapeMovie: universalScraper, scrapeShow: universalScraper, }); diff --git a/src/providers/sources/zoechip/index.ts b/src/providers/sources/zoechip/index.ts index ab4d00f..6b7aae2 100644 --- a/src/providers/sources/zoechip/index.ts +++ b/src/providers/sources/zoechip/index.ts @@ -7,7 +7,7 @@ export const zoechipScraper = makeSourcerer({ id: 'zoechip', name: 'ZoeChip', rank: 200, - flags: [flags.NO_CORS], + flags: [flags.CORS_ALLOWED], scrapeMovie, scrapeShow, }); From d44320e362ed60b26d917865fb084309ec9320a3 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sun, 24 Dec 2023 19:45:43 +0100 Subject: [PATCH 03/17] Add type checking in build step --- package.json | 2 +- tsconfig.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 7395565..e93259a 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ }, "homepage": "https://providers.docs.movie-web.app/", "scripts": { - "build": "vite build", + "build": "vite build && tsc --noEmit", "test": "vitest run", "test:dev": "ts-node ./src/dev-cli.ts", "test:watch": "vitest", diff --git a/tsconfig.json b/tsconfig.json index a57441a..9ac853e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -17,7 +17,7 @@ } }, "include": ["src"], - "exclude": ["node_modules", "**/__tests__/*"], + "exclude": ["node_modules", "**/__test__"], "ts-node": { "require": ["tsconfig-paths/register"] } From 0affe83d24ce4a7ba5a2182ef2f7b8f17f35bd93 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sun, 24 Dec 2023 19:46:12 +0100 Subject: [PATCH 04/17] Allow embeds and videos to return multiple streams + add identifiers to list returns --- src/__test__/providers/checks.test.ts | 4 +-- src/__test__/runner/list.test.ts | 2 +- src/__test__/runner/meta.test.ts | 2 +- src/__test__/utils/valid.test.ts | 10 +++++++ src/main/runner.ts | 4 +-- src/main/targets.ts | 2 +- src/providers/base.ts | 4 +-- src/providers/captions.ts | 1 + src/providers/embeds/febbox/hls.ts | 15 ++++++----- src/providers/embeds/febbox/mp4.ts | 15 ++++++----- src/providers/embeds/febbox/subtitles.ts | 1 + src/providers/embeds/mixdrop.ts | 27 ++++++++++--------- src/providers/embeds/mp4upload.ts | 21 ++++++++------- src/providers/embeds/smashystream/dued.ts | 15 ++++++----- src/providers/embeds/smashystream/video1.ts | 16 ++++++----- src/providers/embeds/streamsb.ts | 15 ++++++----- src/providers/embeds/upcloud.ts | 16 ++++++----- src/providers/embeds/upstream.ts | 15 ++++++----- src/providers/sources/lookmovie/index.ts | 15 ++++++----- src/providers/sources/remotestream.ts | 30 ++++++++++++--------- src/providers/streams.ts | 2 ++ 21 files changed, 142 insertions(+), 90 deletions(-) diff --git a/src/__test__/providers/checks.test.ts b/src/__test__/providers/checks.test.ts index 7ffa845..1396c9d 100644 --- a/src/__test__/providers/checks.test.ts +++ b/src/__test__/providers/checks.test.ts @@ -1,9 +1,9 @@ import { mockEmbeds, mockSources } from '@/__test__/providerTests'; -import { FeatureMap } from '@/main/targets.ts'; +import { FeatureMap } from '@/main/targets'; import { getProviders } from '@/providers/get'; import { vi, describe, it, expect, afterEach } from 'vitest'; -const mocks = await vi.hoisted(async () => (await import('../providerTests.ts')).makeProviderMocks()); +const mocks = await vi.hoisted(async () => (await import('../providerTests')).makeProviderMocks()); vi.mock('@/providers/all', () => mocks); const features: FeatureMap = { diff --git a/src/__test__/runner/list.test.ts b/src/__test__/runner/list.test.ts index 483485e..922392f 100644 --- a/src/__test__/runner/list.test.ts +++ b/src/__test__/runner/list.test.ts @@ -1,6 +1,6 @@ import { mockEmbeds, mockSources } from '@/__test__/providerTests'; import { makeProviders } from '@/main/builder'; -import { targets } from '@/main/targets.ts'; +import { targets } from '@/main/targets'; import { afterEach, describe, expect, it, vi } from 'vitest'; const mocks = await vi.hoisted(async () => (await import('../providerTests.ts')).makeProviderMocks()); diff --git a/src/__test__/runner/meta.test.ts b/src/__test__/runner/meta.test.ts index 4b0e55e..a5cdbf4 100644 --- a/src/__test__/runner/meta.test.ts +++ b/src/__test__/runner/meta.test.ts @@ -1,6 +1,6 @@ import { mockEmbeds, mockSources } from '@/__test__/providerTests'; import { makeProviders } from '@/main/builder'; -import { targets } from '@/main/targets.ts'; +import { targets } from '@/main/targets'; import { afterEach, describe, expect, it, vi } from 'vitest'; const mocks = await vi.hoisted(async () => (await import('../providerTests.ts')).makeProviderMocks()); diff --git a/src/__test__/utils/valid.test.ts b/src/__test__/utils/valid.test.ts index 540fbf3..eab835d 100644 --- a/src/__test__/utils/valid.test.ts +++ b/src/__test__/utils/valid.test.ts @@ -9,7 +9,9 @@ describe('isValidStream()', () => { it('should pass valid streams', () => { expect(isValidStream({ type: "file", + id: "a", flags: [], + captions: [], qualities: { "1080": { type: "mp4", @@ -19,7 +21,9 @@ describe('isValidStream()', () => { })).toBe(true); expect(isValidStream({ type: "hls", + id: "a", flags: [], + captions: [], playlist: "hello-world" })).toBe(true); }); @@ -27,7 +31,9 @@ describe('isValidStream()', () => { it('should detect empty qualities', () => { expect(isValidStream({ type: "file", + id: "a", flags: [], + captions: [], qualities: {} })).toBe(false); }); @@ -35,7 +41,9 @@ describe('isValidStream()', () => { it('should detect empty stream urls', () => { expect(isValidStream({ type: "file", + id: "a", flags: [], + captions: [], qualities: { "1080": { type: "mp4", @@ -48,7 +56,9 @@ describe('isValidStream()', () => { it('should detect emtpy HLS playlists', () => { expect(isValidStream({ type: "hls", + id: "a", flags: [], + captions: [], playlist: "", })).toBe(false); }); diff --git a/src/main/runner.ts b/src/main/runner.ts index 700dbe4..cea7106 100644 --- a/src/main/runner.ts +++ b/src/main/runner.ts @@ -18,13 +18,13 @@ export type RunOutput = { export type SourceRunOutput = { sourceId: string; - stream?: Stream; + stream: Stream[]; embeds: []; }; export type EmbedRunOutput = { embedId: string; - stream?: Stream; + stream: Stream[]; }; export type ProviderRunnerOptions = { diff --git a/src/main/targets.ts b/src/main/targets.ts index 66d6040..1527272 100644 --- a/src/main/targets.ts +++ b/src/main/targets.ts @@ -47,7 +47,7 @@ export const targetToFeatures: Record = { requires: [], disallowed: [], }, -} as const; +}; export function getTargetFeatures(target: Targets): FeatureMap { return targetToFeatures[target]; diff --git a/src/providers/base.ts b/src/providers/base.ts index 022cd87..d503ecd 100644 --- a/src/providers/base.ts +++ b/src/providers/base.ts @@ -9,7 +9,7 @@ export type SourcererEmbed = { export type SourcererOutput = { embeds: SourcererEmbed[]; - stream?: Stream; + stream?: Stream[]; }; export type Sourcerer = { @@ -27,7 +27,7 @@ export function makeSourcerer(state: Sourcerer): Sourcerer { } export type EmbedOutput = { - stream: Stream; + stream: Stream[]; }; export type Embed = { diff --git a/src/providers/captions.ts b/src/providers/captions.ts index 791bfa6..ce3f398 100644 --- a/src/providers/captions.ts +++ b/src/providers/captions.ts @@ -8,6 +8,7 @@ export type CaptionType = keyof typeof captionTypes; export type Caption = { type: CaptionType; + id: string; // only unique per stream url: string; hasCorsRestrictions: boolean; language: string; diff --git a/src/providers/embeds/febbox/hls.ts b/src/providers/embeds/febbox/hls.ts index e0d48e0..78cd4ea 100644 --- a/src/providers/embeds/febbox/hls.ts +++ b/src/providers/embeds/febbox/hls.ts @@ -36,12 +36,15 @@ export const febboxHlsScraper = makeEmbed({ ctx.progress(70); return { - stream: { - type: 'hls', - flags: [flags.CORS_ALLOWED], - captions: await getSubtitles(ctx, id, firstStream.fid, type as MediaTypes, season, episode), - playlist: `https://www.febbox.com/hls/main/${firstStream.oss_fid}.m3u8`, - }, + stream: [ + { + id: 'primary', + type: 'hls', + flags: [flags.CORS_ALLOWED], + captions: await getSubtitles(ctx, id, firstStream.fid, type as MediaTypes, season, episode), + playlist: `https://www.febbox.com/hls/main/${firstStream.oss_fid}.m3u8`, + }, + ], }; }, }); diff --git a/src/providers/embeds/febbox/mp4.ts b/src/providers/embeds/febbox/mp4.ts index 520a35e..bcebbff 100644 --- a/src/providers/embeds/febbox/mp4.ts +++ b/src/providers/embeds/febbox/mp4.ts @@ -39,12 +39,15 @@ export const febboxMp4Scraper = makeEmbed({ ctx.progress(70); return { - stream: { - captions: await getSubtitles(ctx, id, fid, type, episode, season), - qualities, - type: 'file', - flags: [flags.CORS_ALLOWED], - }, + stream: [ + { + id: 'primary', + captions: await getSubtitles(ctx, id, fid, type, episode, season), + qualities, + type: 'file', + flags: [flags.CORS_ALLOWED], + }, + ], }; }, }); diff --git a/src/providers/embeds/febbox/subtitles.ts b/src/providers/embeds/febbox/subtitles.ts index a0394ec..b1b3064 100644 --- a/src/providers/embeds/febbox/subtitles.ts +++ b/src/providers/embeds/febbox/subtitles.ts @@ -54,6 +54,7 @@ export async function getSubtitles( if (!validCode) return; output.push({ + id: subtitleFilePath, language: subtitle.lang, hasCorsRestrictions: true, type: subtitleType, diff --git a/src/providers/embeds/mixdrop.ts b/src/providers/embeds/mixdrop.ts index 007738a..71fd0ac 100644 --- a/src/providers/embeds/mixdrop.ts +++ b/src/providers/embeds/mixdrop.ts @@ -33,21 +33,24 @@ export const mixdropScraper = makeEmbed({ const url = link[1]; return { - stream: { - type: 'file', - flags: [], - captions: [], - qualities: { - unknown: { - type: 'mp4', - url: url.startsWith('http') ? url : `https:${url}`, // URLs don't always start with the protocol - headers: { - // MixDrop requires this header on all streams - Referer: 'https://mixdrop.co/', + stream: [ + { + id: 'primary', + type: 'file', + flags: [], + captions: [], + qualities: { + unknown: { + type: 'mp4', + url: url.startsWith('http') ? url : `https:${url}`, // URLs don't always start with the protocol + headers: { + // MixDrop requires this header on all streams + Referer: 'https://mixdrop.co/', + }, }, }, }, - }, + ], }; }, }); diff --git a/src/providers/embeds/mp4upload.ts b/src/providers/embeds/mp4upload.ts index 72aef8a..69547d8 100644 --- a/src/providers/embeds/mp4upload.ts +++ b/src/providers/embeds/mp4upload.ts @@ -15,17 +15,20 @@ export const mp4uploadScraper = makeEmbed({ if (!streamUrl) throw new Error('Stream url not found in embed code'); return { - stream: { - type: 'file', - flags: [flags.CORS_ALLOWED], - captions: [], - qualities: { - '1080': { - type: 'mp4', - url: streamUrl, + stream: [ + { + id: 'primary', + type: 'file', + flags: [flags.CORS_ALLOWED], + captions: [], + qualities: { + '1080': { + type: 'mp4', + url: streamUrl, + }, }, }, - }, + ], }; }, }); diff --git a/src/providers/embeds/smashystream/dued.ts b/src/providers/embeds/smashystream/dued.ts index df45af2..a143cbc 100644 --- a/src/providers/embeds/smashystream/dued.ts +++ b/src/providers/embeds/smashystream/dued.ts @@ -57,12 +57,15 @@ export const smashyStreamDScraper = makeEmbed({ ); return { - stream: { - playlist: playlistRes, - type: 'hls', - flags: [flags.CORS_ALLOWED], - captions: [], - }, + stream: [ + { + id: 'primary', + playlist: playlistRes, + type: 'hls', + flags: [flags.CORS_ALLOWED], + captions: [], + }, + ], }; }, }); diff --git a/src/providers/embeds/smashystream/video1.ts b/src/providers/embeds/smashystream/video1.ts index 61172fc..391e67c 100644 --- a/src/providers/embeds/smashystream/video1.ts +++ b/src/providers/embeds/smashystream/video1.ts @@ -30,6 +30,7 @@ export const smashyStreamFScraper = makeEmbed({ const captionType = getCaptionTypeFromUrl(url); if (!languageCode || !captionType) return null; return { + id: url, url: url.replace(',', ''), language: languageCode, type: captionType, @@ -42,12 +43,15 @@ export const smashyStreamFScraper = makeEmbed({ .filter((x): x is Caption => x !== null) ?? []; return { - stream: { - playlist: res.sourceUrls[0], - type: 'hls', - flags: [flags.CORS_ALLOWED], - captions, - }, + stream: [ + { + id: 'primary', + playlist: res.sourceUrls[0], + type: 'hls', + flags: [flags.CORS_ALLOWED], + captions, + }, + ], }; }, }); diff --git a/src/providers/embeds/streamsb.ts b/src/providers/embeds/streamsb.ts index 3b0cb7d..98954e5 100644 --- a/src/providers/embeds/streamsb.ts +++ b/src/providers/embeds/streamsb.ts @@ -155,12 +155,15 @@ export const streamsbScraper = makeEmbed({ }, {} as Record); return { - stream: { - type: 'file', - flags: [flags.CORS_ALLOWED], - qualities, - captions: [], - }, + stream: [ + { + id: 'primary', + type: 'file', + flags: [flags.CORS_ALLOWED], + qualities, + captions: [], + }, + ], }; }, }); diff --git a/src/providers/embeds/upcloud.ts b/src/providers/embeds/upcloud.ts index 514d4f1..44e39af 100644 --- a/src/providers/embeds/upcloud.ts +++ b/src/providers/embeds/upcloud.ts @@ -110,6 +110,7 @@ export const upcloudScraper = makeEmbed({ const language = labelToLanguageCode(track.label); if (!language) return; captions.push({ + id: track.file, language, hasCorsRestrictions: false, type, @@ -118,12 +119,15 @@ export const upcloudScraper = makeEmbed({ }); return { - stream: { - type: 'hls', - playlist: sources.file, - flags: [flags.CORS_ALLOWED], - captions, - }, + stream: [ + { + id: 'primary', + type: 'hls', + playlist: sources.file, + flags: [flags.CORS_ALLOWED], + captions, + }, + ], }; }, }); diff --git a/src/providers/embeds/upstream.ts b/src/providers/embeds/upstream.ts index 2b62ee9..ba25056 100644 --- a/src/providers/embeds/upstream.ts +++ b/src/providers/embeds/upstream.ts @@ -21,12 +21,15 @@ export const upstreamScraper = makeEmbed({ if (link) { return { - stream: { - type: 'hls', - playlist: link[1], - flags: [flags.CORS_ALLOWED], - captions: [], - }, + stream: [ + { + id: 'primary', + type: 'hls', + playlist: link[1], + flags: [flags.CORS_ALLOWED], + captions: [], + }, + ], }; } } diff --git a/src/providers/sources/lookmovie/index.ts b/src/providers/sources/lookmovie/index.ts index 08e8e64..606a6e3 100644 --- a/src/providers/sources/lookmovie/index.ts +++ b/src/providers/sources/lookmovie/index.ts @@ -17,12 +17,15 @@ async function universalScraper(ctx: MovieScrapeContext | ShowScrapeContext): Pr return { embeds: [], - stream: { - playlist: videoUrl, - type: 'hls', - flags: [flags.IP_LOCKED], - captions: [], - }, + stream: [ + { + id: 'primary', + playlist: videoUrl, + type: 'hls', + flags: [flags.IP_LOCKED], + captions: [], + }, + ], }; } diff --git a/src/providers/sources/remotestream.ts b/src/providers/sources/remotestream.ts index f3ea478..f97cf4d 100644 --- a/src/providers/sources/remotestream.ts +++ b/src/providers/sources/remotestream.ts @@ -22,12 +22,15 @@ export const remotestreamScraper = makeSourcerer({ return { embeds: [], - stream: { - captions: [], - playlist: playlistLink, - type: 'hls', - flags: [flags.CORS_ALLOWED], - }, + stream: [ + { + id: 'primary', + captions: [], + playlist: playlistLink, + type: 'hls', + flags: [flags.CORS_ALLOWED], + }, + ], }; }, async scrapeMovie(ctx) { @@ -40,12 +43,15 @@ export const remotestreamScraper = makeSourcerer({ return { embeds: [], - stream: { - captions: [], - playlist: playlistLink, - type: 'hls', - flags: [flags.CORS_ALLOWED], - }, + stream: [ + { + id: 'primary', + captions: [], + playlist: playlistLink, + type: 'hls', + flags: [flags.CORS_ALLOWED], + }, + ], }; }, }); diff --git a/src/providers/streams.ts b/src/providers/streams.ts index 1ba4c9a..9a9a9be 100644 --- a/src/providers/streams.ts +++ b/src/providers/streams.ts @@ -11,6 +11,7 @@ export type Qualities = 'unknown' | '360' | '480' | '720' | '1080' | '4k'; export type FileBasedStream = { type: 'file'; + id: string; // only unique per output flags: Flags[]; qualities: Partial>; captions: Caption[]; @@ -18,6 +19,7 @@ export type FileBasedStream = { export type HlsBasedStream = { type: 'hls'; + id: string; // only unique per output flags: Flags[]; playlist: string; captions: Caption[]; From e5989ffbb0d72dec9bca50ec41c31b2cd94de518 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sun, 24 Dec 2023 19:48:27 +0100 Subject: [PATCH 05/17] Export more useful types --- src/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/index.ts b/src/index.ts index ea8b255..d3e310d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,6 @@ export type { EmbedOutput, SourcererOutput } from '@/providers/base'; +export type { Stream, StreamFile, FileBasedStream, HlsBasedStream, Qualities } from '@/providers/streams'; +export type { Fetcher, FetcherOptions } from '@/fetchers/types'; export type { RunOutput } from '@/main/runner'; export type { MetaOutput } from '@/main/meta'; export type { FullScraperEvents } from '@/main/events'; From b70d9aaaf71a483e9618d7a7261e01eb5c7d1d9f Mon Sep 17 00:00:00 2001 From: mrjvs Date: Mon, 25 Dec 2023 00:23:21 +0100 Subject: [PATCH 06/17] Make runners compatible with multi stream output --- src/main/individualRunner.ts | 17 +++++++++++------ src/main/runner.ts | 25 ++++++++++++++----------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/main/individualRunner.ts b/src/main/individualRunner.ts index ac563ea..3b48a29 100644 --- a/src/main/individualRunner.ts +++ b/src/main/individualRunner.ts @@ -50,12 +50,16 @@ export async function scrapeInvidualSource( media: ops.media, }); - // stream doesn't satisfy the feature flags, so gets removed in output - if (output?.stream && (!isValidStream(output.stream) || !flagsAllowedInFeatures(ops.features, output.stream.flags))) { - output.stream = undefined; + // filter output with only valid streams + if (output?.stream) { + output.stream = output.stream + .filter((stream) => isValidStream(stream)) + .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags)); } if (!output) throw new Error('output is null'); + if ((!output.stream || output.stream.length === 0) && output.embeds.length === 0) + throw new NotFoundError('No streams found'); return output; } @@ -88,9 +92,10 @@ export async function scrapeIndividualEmbed( }, }); - if (!isValidStream(output.stream)) throw new NotFoundError('stream is incomplete'); - if (!flagsAllowedInFeatures(ops.features, output.stream.flags)) - throw new NotFoundError("stream doesn't satisfy target feature flags"); + output.stream = output.stream + .filter((stream) => isValidStream(stream)) + .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags)); + if (output.stream.length === 0) throw new NotFoundError('No streams found'); return output; } diff --git a/src/main/runner.ts b/src/main/runner.ts index cea7106..cf9fc82 100644 --- a/src/main/runner.ts +++ b/src/main/runner.ts @@ -80,12 +80,14 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt ...contextBase, media: ops.media, }); - if (output?.stream && !isValidStream(output?.stream)) { - throw new NotFoundError('stream is incomplete'); - } - if (output?.stream && !flagsAllowedInFeatures(ops.features, output.stream.flags)) { - throw new NotFoundError("stream doesn't satisfy target feature flags"); + if (output) { + output.stream = (output.stream ?? []) + .filter((stream) => isValidStream(stream)) + .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags)); } + if (!output) throw Error('No output'); + if ((!output.stream || output.stream.length === 0) && output.embeds.length === 0) + throw new NotFoundError('No streams found'); } catch (err) { if (err instanceof NotFoundError) { ops.events?.update?.({ @@ -107,10 +109,10 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt if (!output) throw new Error('Invalid media type'); // return stream is there are any - if (output.stream) { + if (output.stream?.[0]) { return { sourceId: s.id, - stream: output.stream, + stream: output.stream[0], }; } @@ -144,9 +146,10 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt ...contextBase, url: e.url, }); - if (!flagsAllowedInFeatures(ops.features, embedOutput.stream.flags)) { - throw new NotFoundError("stream doesn't satisfy target feature flags"); - } + embedOutput.stream = embedOutput.stream + .filter((stream) => isValidStream(stream)) + .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags)); + if (embedOutput.stream.length === 0) throw new NotFoundError('No streams found'); } catch (err) { if (err instanceof NotFoundError) { ops.events?.update?.({ @@ -169,7 +172,7 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt return { sourceId: s.id, embedId: scraper.id, - stream: embedOutput.stream, + stream: embedOutput.stream[0], }; } } From 4a557b8140bf8cc5f8d3e3dc81a4b6454f13d7ef Mon Sep 17 00:00:00 2001 From: mrjvs Date: Mon, 25 Dec 2023 01:00:19 +0100 Subject: [PATCH 07/17] Add builder for adding custom sources --- .eslintrc.js | 2 +- src/__test__/providers/checks.test.ts | 38 ++++++-- src/__test__/runner/list.test.ts | 4 +- src/__test__/runner/meta.test.ts | 4 +- src/__test__/utils/features.test.ts | 2 +- src/entrypoint/builder.ts | 93 +++++++++++++++++++ .../builder.ts => entrypoint/controls.ts} | 43 ++++----- src/entrypoint/declare.ts | 37 ++++++++ src/entrypoint/providers.ts | 10 ++ src/{main => entrypoint/utils}/events.ts | 0 src/{main => entrypoint/utils}/media.ts | 0 src/{main => entrypoint/utils}/meta.ts | 2 +- src/{main => entrypoint/utils}/targets.ts | 6 +- src/index.ts | 26 +++--- src/providers/base.ts | 2 +- src/providers/embeds/febbox/common.ts | 2 +- src/providers/embeds/febbox/fileList.ts | 2 +- src/providers/embeds/febbox/hls.ts | 4 +- src/providers/embeds/febbox/mp4.ts | 2 +- src/providers/embeds/mp4upload.ts | 2 +- src/providers/embeds/smashystream/dued.ts | 2 +- src/providers/embeds/smashystream/video1.ts | 2 +- src/providers/embeds/streamsb.ts | 2 +- src/providers/embeds/upcloud.ts | 2 +- src/providers/embeds/upstream.ts | 2 +- src/providers/get.ts | 9 +- src/providers/sources/flixhq/index.ts | 2 +- src/providers/sources/flixhq/scrape.ts | 2 +- src/providers/sources/flixhq/search.ts | 2 +- src/providers/sources/gomovies/index.ts | 2 +- src/providers/sources/kissasian/index.ts | 2 +- src/providers/sources/lookmovie/index.ts | 2 +- src/providers/sources/lookmovie/type.ts | 2 +- src/providers/sources/lookmovie/util.ts | 2 +- src/providers/sources/lookmovie/video.ts | 2 +- src/providers/sources/remotestream.ts | 2 +- src/providers/sources/showbox/index.ts | 2 +- src/providers/sources/smashystream/index.ts | 2 +- src/providers/sources/zoechip/index.ts | 2 +- src/providers/sources/zoechip/scrape.ts | 2 +- src/providers/sources/zoechip/search.ts | 2 +- src/providers/streams.ts | 2 +- src/{main => runners}/individualRunner.ts | 6 +- src/{main => runners}/runner.ts | 6 +- src/utils/compare.ts | 2 +- src/utils/context.ts | 2 +- 46 files changed, 251 insertions(+), 97 deletions(-) create mode 100644 src/entrypoint/builder.ts rename src/{main/builder.ts => entrypoint/controls.ts} (68%) create mode 100644 src/entrypoint/declare.ts create mode 100644 src/entrypoint/providers.ts rename src/{main => entrypoint/utils}/events.ts (100%) rename src/{main => entrypoint/utils}/media.ts (100%) rename src/{main => entrypoint/utils}/meta.ts (96%) rename src/{main => entrypoint/utils}/targets.ts (85%) rename src/{main => runners}/individualRunner.ts (93%) rename src/{main => runners}/runner.ts (96%) diff --git a/.eslintrc.js b/.eslintrc.js index 7939452..0e7322b 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -3,7 +3,7 @@ module.exports = { browser: true, }, extends: ['airbnb-base', 'plugin:@typescript-eslint/recommended', 'plugin:prettier/recommended'], - ignorePatterns: ['lib/*', 'tests/*', '/*.js', '/*.ts', '/**/*.test.ts', 'test/*'], + ignorePatterns: ['lib/*', 'tests/*', '/*.js', '/*.ts', '/src/__test__/*', '/**/*.test.ts', 'test/*'], parser: '@typescript-eslint/parser', parserOptions: { project: './tsconfig.json', diff --git a/src/__test__/providers/checks.test.ts b/src/__test__/providers/checks.test.ts index 1396c9d..404fb31 100644 --- a/src/__test__/providers/checks.test.ts +++ b/src/__test__/providers/checks.test.ts @@ -1,5 +1,6 @@ import { mockEmbeds, mockSources } from '@/__test__/providerTests'; -import { FeatureMap } from '@/main/targets'; +import { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers'; +import { FeatureMap } from '@/entrypoint/utils/targets'; import { getProviders } from '@/providers/get'; import { vi, describe, it, expect, afterEach } from 'vitest'; @@ -19,7 +20,10 @@ describe('getProviders()', () => { it('should return providers', () => { mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD]); mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceB]); - expect(getProviders(features)).toEqual({ + expect(getProviders(features, { + embeds: getBuiltinEmbeds(), + sources: getBuiltinSources(), + })).toEqual({ sources: [mockSources.sourceA, mockSources.sourceB], embeds: [mockEmbeds.embedD], }); @@ -28,7 +32,10 @@ describe('getProviders()', () => { it('should filter out disabled providers', () => { mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedEDisabled]); mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceCDisabled, mockSources.sourceB]); - expect(getProviders(features)).toEqual({ + expect(getProviders(features,{ + embeds: getBuiltinEmbeds(), + sources: getBuiltinSources(), + })).toEqual({ sources: [mockSources.sourceA, mockSources.sourceB], embeds: [mockEmbeds.embedD], }); @@ -37,31 +44,46 @@ describe('getProviders()', () => { it('should throw on duplicate ids in sources', () => { mocks.gatherAllEmbeds.mockReturnValue([]); mocks.gatherAllSources.mockReturnValue([mockSources.sourceAHigherRank, mockSources.sourceA, mockSources.sourceB]); - expect(() => getProviders(features)).toThrowError(); + expect(() => getProviders(features,{ + embeds: getBuiltinEmbeds(), + sources: getBuiltinSources(), + })).toThrowError(); }); it('should throw on duplicate ids in embeds', () => { mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedDHigherRank, mockEmbeds.embedA]); mocks.gatherAllSources.mockReturnValue([]); - expect(() => getProviders(features)).toThrowError(); + expect(() => getProviders(features,{ + embeds: getBuiltinEmbeds(), + sources: getBuiltinSources(), + })).toThrowError(); }); it('should throw on duplicate ids between sources and embeds', () => { mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedA]); mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceB]); - expect(() => getProviders(features)).toThrowError(); + expect(() => getProviders(features,{ + embeds: getBuiltinEmbeds(), + sources: getBuiltinSources(), + })).toThrowError(); }); it('should throw on duplicate rank between sources and embeds', () => { mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedA]); mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceB]); - expect(() => getProviders(features)).toThrowError(); + expect(() => getProviders(features,{ + embeds: getBuiltinEmbeds(), + sources: getBuiltinSources(), + })).toThrowError(); }); it('should not throw with same rank between sources and embeds', () => { mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedHSameRankAsSourceA]); mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceB]); - expect(getProviders(features)).toEqual({ + expect(getProviders(features,{ + embeds: getBuiltinEmbeds(), + sources: getBuiltinSources(), + })).toEqual({ sources: [mockSources.sourceA, mockSources.sourceB], embeds: [mockEmbeds.embedD, mockEmbeds.embedHSameRankAsSourceA], }); diff --git a/src/__test__/runner/list.test.ts b/src/__test__/runner/list.test.ts index 922392f..336de44 100644 --- a/src/__test__/runner/list.test.ts +++ b/src/__test__/runner/list.test.ts @@ -1,6 +1,6 @@ import { mockEmbeds, mockSources } from '@/__test__/providerTests'; -import { makeProviders } from '@/main/builder'; -import { targets } from '@/main/targets'; +import { makeProviders } from '@/entrypoint/declare'; +import { targets } from '@/entrypoint/utils/targets'; import { afterEach, describe, expect, it, vi } from 'vitest'; const mocks = await vi.hoisted(async () => (await import('../providerTests.ts')).makeProviderMocks()); diff --git a/src/__test__/runner/meta.test.ts b/src/__test__/runner/meta.test.ts index a5cdbf4..423a8e6 100644 --- a/src/__test__/runner/meta.test.ts +++ b/src/__test__/runner/meta.test.ts @@ -1,6 +1,6 @@ import { mockEmbeds, mockSources } from '@/__test__/providerTests'; -import { makeProviders } from '@/main/builder'; -import { targets } from '@/main/targets'; +import { makeProviders } from '@/entrypoint/declare'; +import { targets } from '@/entrypoint/utils/targets'; import { afterEach, describe, expect, it, vi } from 'vitest'; const mocks = await vi.hoisted(async () => (await import('../providerTests.ts')).makeProviderMocks()); diff --git a/src/__test__/utils/features.test.ts b/src/__test__/utils/features.test.ts index 0cb6a4e..0da4019 100644 --- a/src/__test__/utils/features.test.ts +++ b/src/__test__/utils/features.test.ts @@ -1,4 +1,4 @@ -import { FeatureMap, Flags, flags, flagsAllowedInFeatures } from "@/main/targets"; +import { FeatureMap, Flags, flags, flagsAllowedInFeatures } from "@/entrypoint/utils/targets"; import { describe, it, expect } from "vitest"; describe('flagsAllowedInFeatures()', () => { diff --git a/src/entrypoint/builder.ts b/src/entrypoint/builder.ts new file mode 100644 index 0000000..abf8288 --- /dev/null +++ b/src/entrypoint/builder.ts @@ -0,0 +1,93 @@ +import { ProviderControls, makeControls } from '@/entrypoint/controls'; +import { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers'; +import { Targets, getTargetFeatures } from '@/entrypoint/utils/targets'; +import { Fetcher } from '@/fetchers/types'; +import { Embed, Sourcerer } from '@/providers/base'; +import { getProviders } from '@/providers/get'; + +export type ProviderBuilder = { + setTarget(target: Targets): ProviderBuilder; + setFetcher(fetcher: Fetcher): ProviderBuilder; + setProxiedFetcher(fetcher: Fetcher): ProviderBuilder; + addSource(scraper: Sourcerer): ProviderBuilder; + addSource(name: string): ProviderBuilder; + addEmbed(scraper: Embed): ProviderBuilder; + addEmbed(name: string): ProviderBuilder; + addBuiltinProviders(): ProviderBuilder; + enableConsistentIpForRequests(): ProviderBuilder; + build(): ProviderControls; +}; + +export function buildProviders(): ProviderBuilder { + let consistentIpForRequests = false; + let target: Targets | null = null; + let fetcher: Fetcher | null = null; + let proxiedFetcher: Fetcher | null = null; + const embeds: Embed[] = []; + const sources: Sourcerer[] = []; + const builtinSources = getBuiltinSources(); + const builtinEmbeds = getBuiltinEmbeds(); + + return { + enableConsistentIpForRequests() { + consistentIpForRequests = true; + return this; + }, + setFetcher(f) { + fetcher = f; + return this; + }, + setProxiedFetcher(f) { + proxiedFetcher = f; + return this; + }, + setTarget(t) { + target = t; + return this; + }, + addSource(input) { + if (typeof input !== 'string') { + sources.push(input); + return this; + } + + const matchingSource = builtinSources.find((v) => v.id === input); + if (!matchingSource) throw new Error('Source not found'); + sources.push(matchingSource); + return this; + }, + addEmbed(input) { + if (typeof input !== 'string') { + embeds.push(input); + return this; + } + + const matchingEmbed = builtinEmbeds.find((v) => v.id === input); + if (!matchingEmbed) throw new Error('Embed not found'); + embeds.push(matchingEmbed); + return this; + }, + addBuiltinProviders() { + sources.push(...builtinSources); + embeds.push(...builtinEmbeds); + return this; + }, + build() { + if (!target) throw new Error('Target not set'); + if (!fetcher) throw new Error('Fetcher not set'); + const features = getTargetFeatures(target, consistentIpForRequests); + const list = getProviders(features, { + embeds, + sources, + }); + + return makeControls({ + fetcher, + proxiedFetcher: proxiedFetcher ?? undefined, + embeds: list.embeds, + sources: list.sources, + features, + }); + }, + }; +} diff --git a/src/main/builder.ts b/src/entrypoint/controls.ts similarity index 68% rename from src/main/builder.ts rename to src/entrypoint/controls.ts index 2ad6911..babcde4 100644 --- a/src/main/builder.ts +++ b/src/entrypoint/controls.ts @@ -1,28 +1,19 @@ +import { FullScraperEvents, IndividualScraperEvents } from '@/entrypoint/utils/events'; +import { ScrapeMedia } from '@/entrypoint/utils/media'; +import { MetaOutput, getAllEmbedMetaSorted, getAllSourceMetaSorted, getSpecificId } from '@/entrypoint/utils/meta'; +import { FeatureMap } from '@/entrypoint/utils/targets'; import { makeFullFetcher } from '@/fetchers/common'; import { Fetcher } from '@/fetchers/types'; -import { FullScraperEvents, IndividualScraperEvents } from '@/main/events'; -import { scrapeIndividualEmbed, scrapeInvidualSource } from '@/main/individualRunner'; -import { ScrapeMedia } from '@/main/media'; -import { MetaOutput, getAllEmbedMetaSorted, getAllSourceMetaSorted, getSpecificId } from '@/main/meta'; -import { RunOutput, runAllProviders } from '@/main/runner'; -import { Targets, flags, getTargetFeatures } from '@/main/targets'; -import { EmbedOutput, SourcererOutput } from '@/providers/base'; -import { getProviders } from '@/providers/get'; +import { Embed, EmbedOutput, Sourcerer, SourcererOutput } from '@/providers/base'; +import { scrapeIndividualEmbed, scrapeInvidualSource } from '@/runners/individualRunner'; +import { RunOutput, runAllProviders } from '@/runners/runner'; -export interface ProviderBuilderOptions { - // fetcher, every web request gets called through here +export interface ProviderControlsInput { fetcher: Fetcher; - - // proxied fetcher, if the scraper needs to access a CORS proxy. this fetcher will be called instead - // of the normal fetcher. Defaults to the normal fetcher. proxiedFetcher?: Fetcher; - - // target of where the streams will be used - target: Targets; - - // Set this to true, if the requests will have the same IP as - // the device that the stream will be played on - consistentIpForRequests?: boolean; + features: FeatureMap; + sources: Sourcerer[]; + embeds: Embed[]; } export interface RunnerOptions { @@ -84,12 +75,14 @@ export interface ProviderControls { listEmbeds(): MetaOutput[]; } -export function makeProviders(ops: ProviderBuilderOptions): ProviderControls { - const features = getTargetFeatures(ops.target); - if (!ops.consistentIpForRequests) features.disallowed.push(flags.IP_LOCKED); - const list = getProviders(features); +export function makeControls(ops: ProviderControlsInput): ProviderControls { + const list = { + embeds: ops.embeds, + sources: ops.sources, + }; + const providerRunnerOps = { - features, + features: ops.features, fetcher: makeFullFetcher(ops.fetcher), proxiedFetcher: makeFullFetcher(ops.proxiedFetcher ?? ops.fetcher), }; diff --git a/src/entrypoint/declare.ts b/src/entrypoint/declare.ts new file mode 100644 index 0000000..152ee87 --- /dev/null +++ b/src/entrypoint/declare.ts @@ -0,0 +1,37 @@ +import { makeControls } from '@/entrypoint/controls'; +import { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers'; +import { Targets, getTargetFeatures } from '@/entrypoint/utils/targets'; +import { Fetcher } from '@/fetchers/types'; +import { getProviders } from '@/providers/get'; + +export interface ProviderMakerOptions { + // fetcher, every web request gets called through here + fetcher: Fetcher; + + // proxied fetcher, if the scraper needs to access a CORS proxy. this fetcher will be called instead + // of the normal fetcher. Defaults to the normal fetcher. + proxiedFetcher?: Fetcher; + + // target of where the streams will be used + target: Targets; + + // Set this to true, if the requests will have the same IP as + // the device that the stream will be played on + consistentIpForRequests?: boolean; +} + +export function makeProviders(ops: ProviderMakerOptions) { + const features = getTargetFeatures(ops.target, ops.consistentIpForRequests ?? false); + const list = getProviders(features, { + embeds: getBuiltinEmbeds(), + sources: getBuiltinSources(), + }); + + return makeControls({ + embeds: list.embeds, + sources: list.sources, + features, + fetcher: ops.fetcher, + proxiedFetcher: ops.proxiedFetcher, + }); +} diff --git a/src/entrypoint/providers.ts b/src/entrypoint/providers.ts new file mode 100644 index 0000000..b306417 --- /dev/null +++ b/src/entrypoint/providers.ts @@ -0,0 +1,10 @@ +import { gatherAllEmbeds, gatherAllSources } from '@/providers/all'; +import { Embed, Sourcerer } from '@/providers/base'; + +export function getBuiltinSources(): Sourcerer[] { + return gatherAllSources(); +} + +export function getBuiltinEmbeds(): Embed[] { + return gatherAllEmbeds(); +} diff --git a/src/main/events.ts b/src/entrypoint/utils/events.ts similarity index 100% rename from src/main/events.ts rename to src/entrypoint/utils/events.ts diff --git a/src/main/media.ts b/src/entrypoint/utils/media.ts similarity index 100% rename from src/main/media.ts rename to src/entrypoint/utils/media.ts diff --git a/src/main/meta.ts b/src/entrypoint/utils/meta.ts similarity index 96% rename from src/main/meta.ts rename to src/entrypoint/utils/meta.ts index 5696183..5e54b2a 100644 --- a/src/main/meta.ts +++ b/src/entrypoint/utils/meta.ts @@ -1,4 +1,4 @@ -import { MediaTypes } from '@/main/media'; +import { MediaTypes } from '@/entrypoint/utils/media'; import { Embed, Sourcerer } from '@/providers/base'; import { ProviderList } from '@/providers/get'; diff --git a/src/main/targets.ts b/src/entrypoint/utils/targets.ts similarity index 85% rename from src/main/targets.ts rename to src/entrypoint/utils/targets.ts index 1527272..16a02da 100644 --- a/src/main/targets.ts +++ b/src/entrypoint/utils/targets.ts @@ -49,8 +49,10 @@ export const targetToFeatures: Record = { }, }; -export function getTargetFeatures(target: Targets): FeatureMap { - return targetToFeatures[target]; +export function getTargetFeatures(target: Targets, consistentIpForRequests: boolean): FeatureMap { + const features = targetToFeatures[target]; + if (!consistentIpForRequests) features.disallowed.push(flags.IP_LOCKED); + return features; } export function flagsAllowedInFeatures(features: FeatureMap, inputFlags: Flags[]): boolean { diff --git a/src/index.ts b/src/index.ts index d3e310d..347f8dc 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,21 +1,19 @@ export type { EmbedOutput, SourcererOutput } from '@/providers/base'; export type { Stream, StreamFile, FileBasedStream, HlsBasedStream, Qualities } from '@/providers/streams'; export type { Fetcher, FetcherOptions } from '@/fetchers/types'; -export type { RunOutput } from '@/main/runner'; -export type { MetaOutput } from '@/main/meta'; -export type { FullScraperEvents } from '@/main/events'; -export type { Targets, Flags } from '@/main/targets'; -export type { MediaTypes, ShowMedia, ScrapeMedia, MovieMedia } from '@/main/media'; -export type { - ProviderBuilderOptions, - ProviderControls, - RunnerOptions, - EmbedRunnerOptions, - SourceRunnerOptions, -} from '@/main/builder'; +export type { RunOutput } from '@/runners/runner'; +export type { MetaOutput } from '@/entrypoint/utils/meta'; +export type { FullScraperEvents } from '@/entrypoint/utils/events'; +export type { Targets, Flags } from '@/entrypoint/utils/targets'; +export type { MediaTypes, ShowMedia, ScrapeMedia, MovieMedia } from '@/entrypoint/utils/media'; +export type { ProviderControls, RunnerOptions, EmbedRunnerOptions, SourceRunnerOptions } from '@/entrypoint/controls'; +export type { ProviderBuilder } from '@/entrypoint/builder'; +export type { ProviderMakerOptions } from '@/entrypoint/declare'; export { NotFoundError } from '@/utils/errors'; -export { makeProviders } from '@/main/builder'; +export { makeProviders } from '@/entrypoint/declare'; +export { buildProviders } from '@/entrypoint/builder'; +export { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers'; export { makeStandardFetcher } from '@/fetchers/standardFetch'; export { makeSimpleProxyFetcher } from '@/fetchers/simpleProxy'; -export { flags, targets } from '@/main/targets'; +export { flags, targets } from '@/entrypoint/utils/targets'; diff --git a/src/providers/base.ts b/src/providers/base.ts index d503ecd..7371166 100644 --- a/src/providers/base.ts +++ b/src/providers/base.ts @@ -1,4 +1,4 @@ -import { Flags } from '@/main/targets'; +import { Flags } from '@/entrypoint/utils/targets'; import { Stream } from '@/providers/streams'; import { EmbedScrapeContext, MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; diff --git a/src/providers/embeds/febbox/common.ts b/src/providers/embeds/febbox/common.ts index 4348c25..5d902db 100644 --- a/src/providers/embeds/febbox/common.ts +++ b/src/providers/embeds/febbox/common.ts @@ -1,4 +1,4 @@ -import { MediaTypes } from '@/main/media'; +import { MediaTypes } from '@/entrypoint/utils/media'; export const febBoxBase = `https://www.febbox.com`; diff --git a/src/providers/embeds/febbox/fileList.ts b/src/providers/embeds/febbox/fileList.ts index b0c03fb..593fc77 100644 --- a/src/providers/embeds/febbox/fileList.ts +++ b/src/providers/embeds/febbox/fileList.ts @@ -1,4 +1,4 @@ -import { MediaTypes } from '@/main/media'; +import { MediaTypes } from '@/entrypoint/utils/media'; import { FebboxFileList, febBoxBase } from '@/providers/embeds/febbox/common'; import { EmbedScrapeContext } from '@/utils/context'; diff --git a/src/providers/embeds/febbox/hls.ts b/src/providers/embeds/febbox/hls.ts index 78cd4ea..d9fa54f 100644 --- a/src/providers/embeds/febbox/hls.ts +++ b/src/providers/embeds/febbox/hls.ts @@ -1,5 +1,5 @@ -import { MediaTypes } from '@/main/media'; -import { flags } from '@/main/targets'; +import { MediaTypes } from '@/entrypoint/utils/media'; +import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; import { parseInputUrl } from '@/providers/embeds/febbox/common'; import { getStreams } from '@/providers/embeds/febbox/fileList'; diff --git a/src/providers/embeds/febbox/mp4.ts b/src/providers/embeds/febbox/mp4.ts index bcebbff..1122e53 100644 --- a/src/providers/embeds/febbox/mp4.ts +++ b/src/providers/embeds/febbox/mp4.ts @@ -1,4 +1,4 @@ -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; import { parseInputUrl } from '@/providers/embeds/febbox/common'; import { getStreamQualities } from '@/providers/embeds/febbox/qualities'; diff --git a/src/providers/embeds/mp4upload.ts b/src/providers/embeds/mp4upload.ts index 69547d8..5bc8576 100644 --- a/src/providers/embeds/mp4upload.ts +++ b/src/providers/embeds/mp4upload.ts @@ -1,4 +1,4 @@ -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; export const mp4uploadScraper = makeEmbed({ diff --git a/src/providers/embeds/smashystream/dued.ts b/src/providers/embeds/smashystream/dued.ts index a143cbc..c7a1d1d 100644 --- a/src/providers/embeds/smashystream/dued.ts +++ b/src/providers/embeds/smashystream/dued.ts @@ -1,6 +1,6 @@ import { load } from 'cheerio'; -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; type DPlayerSourcesResponse = { diff --git a/src/providers/embeds/smashystream/video1.ts b/src/providers/embeds/smashystream/video1.ts index 391e67c..1fc2edf 100644 --- a/src/providers/embeds/smashystream/video1.ts +++ b/src/providers/embeds/smashystream/video1.ts @@ -1,4 +1,4 @@ -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; diff --git a/src/providers/embeds/streamsb.ts b/src/providers/embeds/streamsb.ts index 98954e5..2314be4 100644 --- a/src/providers/embeds/streamsb.ts +++ b/src/providers/embeds/streamsb.ts @@ -3,7 +3,7 @@ import Base64 from 'crypto-js/enc-base64'; import Utf8 from 'crypto-js/enc-utf8'; import FormData from 'form-data'; -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; import { StreamFile } from '@/providers/streams'; import { EmbedScrapeContext } from '@/utils/context'; diff --git a/src/providers/embeds/upcloud.ts b/src/providers/embeds/upcloud.ts index 44e39af..7880cc4 100644 --- a/src/providers/embeds/upcloud.ts +++ b/src/providers/embeds/upcloud.ts @@ -1,6 +1,6 @@ import crypto from 'crypto-js'; -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; diff --git a/src/providers/embeds/upstream.ts b/src/providers/embeds/upstream.ts index ba25056..8becf22 100644 --- a/src/providers/embeds/upstream.ts +++ b/src/providers/embeds/upstream.ts @@ -1,6 +1,6 @@ import * as unpacker from 'unpacker'; -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; const packedRegex = /(eval\(function\(p,a,c,k,e,d\).*\)\)\))/; diff --git a/src/providers/get.ts b/src/providers/get.ts index 085aba5..900bbd1 100644 --- a/src/providers/get.ts +++ b/src/providers/get.ts @@ -1,5 +1,4 @@ -import { FeatureMap, flagsAllowedInFeatures } from '@/main/targets'; -import { gatherAllEmbeds, gatherAllSources } from '@/providers/all'; +import { FeatureMap, flagsAllowedInFeatures } from '@/entrypoint/utils/targets'; import { Embed, Sourcerer } from '@/providers/base'; import { hasDuplicates } from '@/utils/predicates'; @@ -8,9 +7,9 @@ export interface ProviderList { embeds: Embed[]; } -export function getProviders(features: FeatureMap): ProviderList { - const sources = gatherAllSources().filter((v) => !v?.disabled); - const embeds = gatherAllEmbeds().filter((v) => !v?.disabled); +export function getProviders(features: FeatureMap, list: ProviderList): ProviderList { + const sources = list.sources.filter((v) => !v?.disabled); + const embeds = list.embeds.filter((v) => !v?.disabled); const combined = [...sources, ...embeds]; const anyDuplicateId = hasDuplicates(combined.map((v) => v.id)); diff --git a/src/providers/sources/flixhq/index.ts b/src/providers/sources/flixhq/index.ts index b866677..e9fbaef 100644 --- a/src/providers/sources/flixhq/index.ts +++ b/src/providers/sources/flixhq/index.ts @@ -1,4 +1,4 @@ -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { makeSourcerer } from '@/providers/base'; import { upcloudScraper } from '@/providers/embeds/upcloud'; import { getFlixhqMovieSources, getFlixhqShowSources, getFlixhqSourceDetails } from '@/providers/sources/flixhq/scrape'; diff --git a/src/providers/sources/flixhq/scrape.ts b/src/providers/sources/flixhq/scrape.ts index a73916c..3f42e9a 100644 --- a/src/providers/sources/flixhq/scrape.ts +++ b/src/providers/sources/flixhq/scrape.ts @@ -1,6 +1,6 @@ import { load } from 'cheerio'; -import { MovieMedia, ShowMedia } from '@/main/media'; +import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; import { flixHqBase } from '@/providers/sources/flixhq/common'; import { ScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; diff --git a/src/providers/sources/flixhq/search.ts b/src/providers/sources/flixhq/search.ts index d52f65a..bcab033 100644 --- a/src/providers/sources/flixhq/search.ts +++ b/src/providers/sources/flixhq/search.ts @@ -1,6 +1,6 @@ import { load } from 'cheerio'; -import { MovieMedia, ShowMedia } from '@/main/media'; +import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; import { flixHqBase } from '@/providers/sources/flixhq/common'; import { compareMedia, compareTitle } from '@/utils/compare'; import { ScrapeContext } from '@/utils/context'; diff --git a/src/providers/sources/gomovies/index.ts b/src/providers/sources/gomovies/index.ts index ff43411..9bec3e0 100644 --- a/src/providers/sources/gomovies/index.ts +++ b/src/providers/sources/gomovies/index.ts @@ -1,6 +1,6 @@ import { load } from 'cheerio'; -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { makeSourcerer } from '@/providers/base'; import { upcloudScraper } from '@/providers/embeds/upcloud'; import { NotFoundError } from '@/utils/errors'; diff --git a/src/providers/sources/kissasian/index.ts b/src/providers/sources/kissasian/index.ts index d856ea5..4b8032b 100644 --- a/src/providers/sources/kissasian/index.ts +++ b/src/providers/sources/kissasian/index.ts @@ -1,6 +1,6 @@ import { load } from 'cheerio'; -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { makeSourcerer } from '@/providers/base'; import { NotFoundError } from '@/utils/errors'; diff --git a/src/providers/sources/lookmovie/index.ts b/src/providers/sources/lookmovie/index.ts index 606a6e3..5cd82e9 100644 --- a/src/providers/sources/lookmovie/index.ts +++ b/src/providers/sources/lookmovie/index.ts @@ -1,4 +1,4 @@ -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { SourcererOutput, makeSourcerer } from '@/providers/base'; import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; diff --git a/src/providers/sources/lookmovie/type.ts b/src/providers/sources/lookmovie/type.ts index 3731520..8335c96 100644 --- a/src/providers/sources/lookmovie/type.ts +++ b/src/providers/sources/lookmovie/type.ts @@ -1,4 +1,4 @@ -import { MovieMedia } from '@/main/media'; +import { MovieMedia } from '@/entrypoint/utils/media'; // ! Types interface BaseConfig { diff --git a/src/providers/sources/lookmovie/util.ts b/src/providers/sources/lookmovie/util.ts index 231e715..7c8f202 100644 --- a/src/providers/sources/lookmovie/util.ts +++ b/src/providers/sources/lookmovie/util.ts @@ -1,4 +1,4 @@ -import { MovieMedia, ShowMedia } from '@/main/media'; +import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; import { compareMedia } from '@/utils/compare'; import { ScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; diff --git a/src/providers/sources/lookmovie/video.ts b/src/providers/sources/lookmovie/video.ts index 38ace62..f439229 100644 --- a/src/providers/sources/lookmovie/video.ts +++ b/src/providers/sources/lookmovie/video.ts @@ -1,4 +1,4 @@ -import { MovieMedia, ShowMedia } from '@/main/media'; +import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; import { ScrapeContext } from '@/utils/context'; import { StreamsDataResult } from './type'; diff --git a/src/providers/sources/remotestream.ts b/src/providers/sources/remotestream.ts index f97cf4d..6d0a44c 100644 --- a/src/providers/sources/remotestream.ts +++ b/src/providers/sources/remotestream.ts @@ -1,4 +1,4 @@ -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { makeSourcerer } from '@/providers/base'; import { NotFoundError } from '@/utils/errors'; diff --git a/src/providers/sources/showbox/index.ts b/src/providers/sources/showbox/index.ts index 064011e..267a6ef 100644 --- a/src/providers/sources/showbox/index.ts +++ b/src/providers/sources/showbox/index.ts @@ -1,4 +1,4 @@ -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { SourcererOutput, makeSourcerer } from '@/providers/base'; import { febboxHlsScraper } from '@/providers/embeds/febbox/hls'; import { febboxMp4Scraper } from '@/providers/embeds/febbox/mp4'; diff --git a/src/providers/sources/smashystream/index.ts b/src/providers/sources/smashystream/index.ts index c7601f8..62102ea 100644 --- a/src/providers/sources/smashystream/index.ts +++ b/src/providers/sources/smashystream/index.ts @@ -1,6 +1,6 @@ import { load } from 'cheerio'; -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { SourcererEmbed, SourcererOutput, makeSourcerer } from '@/providers/base'; import { smashyStreamDScraper } from '@/providers/embeds/smashystream/dued'; import { smashyStreamFScraper } from '@/providers/embeds/smashystream/video1'; diff --git a/src/providers/sources/zoechip/index.ts b/src/providers/sources/zoechip/index.ts index 6b7aae2..ee7f3a6 100644 --- a/src/providers/sources/zoechip/index.ts +++ b/src/providers/sources/zoechip/index.ts @@ -1,4 +1,4 @@ -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { makeSourcerer } from '@/providers/base'; import { scrapeMovie } from '@/providers/sources/zoechip/scrape-movie'; import { scrapeShow } from '@/providers/sources/zoechip/scrape-show'; diff --git a/src/providers/sources/zoechip/scrape.ts b/src/providers/sources/zoechip/scrape.ts index 126a3e0..d3eb183 100644 --- a/src/providers/sources/zoechip/scrape.ts +++ b/src/providers/sources/zoechip/scrape.ts @@ -1,6 +1,6 @@ import { load } from 'cheerio'; -import { ShowMedia } from '@/main/media'; +import { ShowMedia } from '@/entrypoint/utils/media'; import { ZoeChipSourceDetails, zoeBase } from '@/providers/sources/zoechip/common'; import { MovieScrapeContext, ScrapeContext, ShowScrapeContext } from '@/utils/context'; diff --git a/src/providers/sources/zoechip/search.ts b/src/providers/sources/zoechip/search.ts index 6297bc4..f3a838d 100644 --- a/src/providers/sources/zoechip/search.ts +++ b/src/providers/sources/zoechip/search.ts @@ -1,6 +1,6 @@ import { load } from 'cheerio'; -import { MovieMedia, ShowMedia } from '@/main/media'; +import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; import { zoeBase } from '@/providers/sources/zoechip/common'; import { compareMedia } from '@/utils/compare'; import { ScrapeContext } from '@/utils/context'; diff --git a/src/providers/streams.ts b/src/providers/streams.ts index 9a9a9be..03ae873 100644 --- a/src/providers/streams.ts +++ b/src/providers/streams.ts @@ -1,4 +1,4 @@ -import { Flags } from '@/main/targets'; +import { Flags } from '@/entrypoint/utils/targets'; import { Caption } from '@/providers/captions'; export type StreamFile = { diff --git a/src/main/individualRunner.ts b/src/runners/individualRunner.ts similarity index 93% rename from src/main/individualRunner.ts rename to src/runners/individualRunner.ts index 3b48a29..2befd84 100644 --- a/src/main/individualRunner.ts +++ b/src/runners/individualRunner.ts @@ -1,7 +1,7 @@ +import { IndividualScraperEvents } from '@/entrypoint/utils/events'; +import { ScrapeMedia } from '@/entrypoint/utils/media'; +import { FeatureMap, flagsAllowedInFeatures } from '@/entrypoint/utils/targets'; import { UseableFetcher } from '@/fetchers/types'; -import { IndividualScraperEvents } from '@/main/events'; -import { ScrapeMedia } from '@/main/media'; -import { FeatureMap, flagsAllowedInFeatures } from '@/main/targets'; import { EmbedOutput, SourcererOutput } from '@/providers/base'; import { ProviderList } from '@/providers/get'; import { ScrapeContext } from '@/utils/context'; diff --git a/src/main/runner.ts b/src/runners/runner.ts similarity index 96% rename from src/main/runner.ts rename to src/runners/runner.ts index cf9fc82..1774213 100644 --- a/src/main/runner.ts +++ b/src/runners/runner.ts @@ -1,7 +1,7 @@ +import { FullScraperEvents } from '@/entrypoint/utils/events'; +import { ScrapeMedia } from '@/entrypoint/utils/media'; +import { FeatureMap, flagsAllowedInFeatures } from '@/entrypoint/utils/targets'; import { UseableFetcher } from '@/fetchers/types'; -import { FullScraperEvents } from '@/main/events'; -import { ScrapeMedia } from '@/main/media'; -import { FeatureMap, flagsAllowedInFeatures } from '@/main/targets'; import { EmbedOutput, SourcererOutput } from '@/providers/base'; import { ProviderList } from '@/providers/get'; import { Stream } from '@/providers/streams'; diff --git a/src/utils/compare.ts b/src/utils/compare.ts index 8cce7da..cceffb8 100644 --- a/src/utils/compare.ts +++ b/src/utils/compare.ts @@ -1,4 +1,4 @@ -import { CommonMedia } from '@/main/media'; +import { CommonMedia } from '@/entrypoint/utils/media'; export function normalizeTitle(title: string): string { return title diff --git a/src/utils/context.ts b/src/utils/context.ts index 1a84253..36cd151 100644 --- a/src/utils/context.ts +++ b/src/utils/context.ts @@ -1,5 +1,5 @@ +import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; import { UseableFetcher } from '@/fetchers/types'; -import { MovieMedia, ShowMedia } from '@/main/media'; export type ScrapeContext = { proxiedFetcher: (...params: Parameters>) => ReturnType>; From 09ba0ebcc186ffad9f0be37497818fbd4ba7dec3 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Mon, 25 Dec 2023 01:18:05 +0100 Subject: [PATCH 08/17] Fix browser integration test --- tests/browser/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/browser/index.ts b/tests/browser/index.ts index 7ad11d1..aab14cc 100644 --- a/tests/browser/index.ts +++ b/tests/browser/index.ts @@ -3,6 +3,6 @@ import { makeProviders, makeStandardFetcher, targets } from '../../lib/index.mjs (window as any).TEST = () => { makeProviders({ fetcher: makeStandardFetcher(fetch), - target: targets.ALL, + target: targets.ANY, }); } From ffe7ae0985b2e7336f0203e32593a205680db266 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 26 Dec 2023 15:43:52 +0100 Subject: [PATCH 09/17] More elaborate fetcher API's --- src/entrypoint/controls.ts | 6 +++--- src/fetchers/common.ts | 8 ++++++-- src/fetchers/fetch.ts | 5 +++++ src/fetchers/simpleProxy.ts | 21 ++++++++++++++++++++- src/fetchers/standardFetch.ts | 26 +++++++++++++++++++++++--- src/fetchers/types.ts | 19 ++++++++++++++++--- 6 files changed, 73 insertions(+), 12 deletions(-) diff --git a/src/entrypoint/controls.ts b/src/entrypoint/controls.ts index babcde4..5ff400b 100644 --- a/src/entrypoint/controls.ts +++ b/src/entrypoint/controls.ts @@ -2,7 +2,7 @@ import { FullScraperEvents, IndividualScraperEvents } from '@/entrypoint/utils/e import { ScrapeMedia } from '@/entrypoint/utils/media'; import { MetaOutput, getAllEmbedMetaSorted, getAllSourceMetaSorted, getSpecificId } from '@/entrypoint/utils/meta'; import { FeatureMap } from '@/entrypoint/utils/targets'; -import { makeFullFetcher } from '@/fetchers/common'; +import { makeFetcher } from '@/fetchers/common'; import { Fetcher } from '@/fetchers/types'; import { Embed, EmbedOutput, Sourcerer, SourcererOutput } from '@/providers/base'; import { scrapeIndividualEmbed, scrapeInvidualSource } from '@/runners/individualRunner'; @@ -83,8 +83,8 @@ export function makeControls(ops: ProviderControlsInput): ProviderControls { const providerRunnerOps = { features: ops.features, - fetcher: makeFullFetcher(ops.fetcher), - proxiedFetcher: makeFullFetcher(ops.proxiedFetcher ?? ops.fetcher), + fetcher: makeFetcher(ops.fetcher), + proxiedFetcher: makeFetcher(ops.proxiedFetcher ?? ops.fetcher), }; return { diff --git a/src/fetchers/common.ts b/src/fetchers/common.ts index e31b6d1..71956ba 100644 --- a/src/fetchers/common.ts +++ b/src/fetchers/common.ts @@ -26,14 +26,18 @@ export function makeFullUrl(url: string, ops?: FullUrlOptions): string { return parsedUrl.toString(); } -export function makeFullFetcher(fetcher: Fetcher): UseableFetcher { - return (url, ops) => { +export function makeFetcher(fetcher: Fetcher): UseableFetcher { + const newFetcher = (url: string, ops?: FetcherOptions) => { return fetcher(url, { headers: ops?.headers ?? {}, method: ops?.method ?? 'GET', query: ops?.query ?? {}, baseUrl: ops?.baseUrl ?? '', + readHeaders: ops?.readHeaders ?? [], body: ops?.body, }); }; + const output: UseableFetcher = async (url, ops) => (await newFetcher(url, ops)).body; + output.full = newFetcher; + return output; } diff --git a/src/fetchers/fetch.ts b/src/fetchers/fetch.ts index 1d419f0..d2156d0 100644 --- a/src/fetchers/fetch.ts +++ b/src/fetchers/fetch.ts @@ -11,12 +11,17 @@ export type FetchOps = { export type FetchHeaders = { get(key: string): string | null; + set(key: string, value: string): void; }; export type FetchReply = { text(): Promise; json(): Promise; + extraHeaders?: FetchHeaders; + extraUrl?: string; headers: FetchHeaders; + url: string; + status: number; }; export type FetchLike = (url: string, ops?: FetchOps | undefined) => Promise; diff --git a/src/fetchers/simpleProxy.ts b/src/fetchers/simpleProxy.ts index 07b048e..21ed5ca 100644 --- a/src/fetchers/simpleProxy.ts +++ b/src/fetchers/simpleProxy.ts @@ -9,9 +9,28 @@ const headerMap: Record = { origin: 'X-Origin', }; +const responseHeaderMap: Record = { + 'x-set-cookie': 'Set-Cookie', +}; + export function makeSimpleProxyFetcher(proxyUrl: string, f: FetchLike): Fetcher { - const fetcher = makeStandardFetcher(f); const proxiedFetch: Fetcher = async (url, ops) => { + const fetcher = makeStandardFetcher(async (a, b) => { + const res = await f(a, b); + + // set extra headers that cant normally be accessed + res.extraHeaders = new Headers(); + Object.entries(responseHeaderMap).forEach((entry) => { + const value = res.headers.get(entry[0]); + if (!value) return; + res.extraHeaders?.set(entry[0].toLowerCase(), value); + }); + + // set correct final url + res.extraUrl = res.headers.get('X-Final-Destination') ?? res.url; + return res; + }); + const fullUrl = makeFullUrl(url, ops); const headerEntries = Object.entries(ops.headers).map((entry) => { diff --git a/src/fetchers/standardFetch.ts b/src/fetchers/standardFetch.ts index dd84893..a17dd92 100644 --- a/src/fetchers/standardFetch.ts +++ b/src/fetchers/standardFetch.ts @@ -1,8 +1,20 @@ import { serializeBody } from '@/fetchers/body'; import { makeFullUrl } from '@/fetchers/common'; -import { FetchLike } from '@/fetchers/fetch'; +import { FetchLike, FetchReply } from '@/fetchers/fetch'; import { Fetcher } from '@/fetchers/types'; +function getHeaders(list: string[], res: FetchReply): Headers { + const output = new Headers(); + list.forEach((header) => { + const realHeader = header.toLowerCase(); + const value = res.headers.get(realHeader); + const extraValue = res.extraHeaders?.get(realHeader); + if (!value) return; + output.set(realHeader, extraValue ?? value); + }); + return output; +} + export function makeStandardFetcher(f: FetchLike): Fetcher { const normalFetch: Fetcher = async (url, ops) => { const fullUrl = makeFullUrl(url, ops); @@ -17,9 +29,17 @@ export function makeStandardFetcher(f: FetchLike): Fetcher { body: seralizedBody.body, }); + let body: any; const isJson = res.headers.get('content-type')?.includes('application/json'); - if (isJson) return res.json(); - return res.text(); + if (isJson) body = await res.json(); + else body = res.text(); + + return { + body, + finalUrl: res.extraUrl ?? res.url, + headers: getHeaders(ops.readHeaders, res), + statusCode: res.status, + }; }; return normalFetch; diff --git a/src/fetchers/types.ts b/src/fetchers/types.ts index 2d14748..4b9cdc0 100644 --- a/src/fetchers/types.ts +++ b/src/fetchers/types.ts @@ -5,22 +5,35 @@ export type FetcherOptions = { headers?: Record; query?: Record; method?: 'GET' | 'POST'; + readHeaders?: string[]; body?: Record | string | FormData | URLSearchParams; }; +// Version of the options that always has the defaults set +// This is to make making fetchers yourself easier export type DefaultedFetcherOptions = { baseUrl?: string; body?: Record | string | FormData; headers: Record; query: Record; + readHeaders: string[]; method: 'GET' | 'POST'; }; -export type Fetcher = { - (url: string, ops: DefaultedFetcherOptions): Promise; +export type FetcherResponse = { + statusCode: number; + headers: Headers; + finalUrl: string; + body: T; }; -// this feature has some quality of life features +// This is the version that will be inputted by library users +export type Fetcher = { + (url: string, ops: DefaultedFetcherOptions): Promise>; +}; + +// This is the version that scrapers will be interacting with export type UseableFetcher = { (url: string, ops?: FetcherOptions): Promise; + full: (url: string, ops?: FetcherOptions) => Promise>; }; From 4eaae64e4afc290a22d41375821f505bcb55be18 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 26 Dec 2023 16:02:25 +0100 Subject: [PATCH 10/17] Fix fetcher tests --- src/__test__/fetchers/simpleProxy.test.ts | 15 ++++++++++++++- src/__test__/fetchers/standard.test.ts | 16 +++++++++++++++- src/fetchers/standardFetch.ts | 2 +- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/__test__/fetchers/simpleProxy.test.ts b/src/__test__/fetchers/simpleProxy.test.ts index 5066e63..fc1137a 100644 --- a/src/__test__/fetchers/simpleProxy.test.ts +++ b/src/__test__/fetchers/simpleProxy.test.ts @@ -16,6 +16,8 @@ describe("makeSimpleProxyFetcher()", () => { headers: new Headers({ "content-type": "text/plain", }), + status: 204, + url: "test123", text() { return Promise.resolve(value); }, @@ -24,6 +26,8 @@ describe("makeSimpleProxyFetcher()", () => { headers: new Headers({ "content-type": "application/json", }), + status: 204, + url: "test123", json() { return Promise.resolve(value); }, @@ -31,7 +35,11 @@ describe("makeSimpleProxyFetcher()", () => { } function expectFetchCall(ops: { inputUrl: string, input: DefaultedFetcherOptions, outputUrl?: string, output: any, outputBody: any }) { - expect(fetcher(ops.inputUrl, ops.input)).resolves.toEqual(ops.outputBody); + const prom = fetcher(ops.inputUrl, ops.input); + expect((async () => (await prom).body)()).resolves.toEqual(ops.outputBody); + expect((async () => (await prom).headers.entries())()).resolves.toEqual((new Headers()).entries()); + expect((async () => (await prom).statusCode)()).resolves.toEqual(204); + expect((async () => (await prom).finalUrl)()).resolves.toEqual("test123"); expect(fetch).toBeCalledWith(ops.outputUrl ?? ops.inputUrl, ops.output); vi.clearAllMocks(); } @@ -43,6 +51,7 @@ describe("makeSimpleProxyFetcher()", () => { input: { method: "GET", query: {}, + readHeaders: [], headers: { "X-Hello": "world", }, @@ -62,6 +71,7 @@ describe("makeSimpleProxyFetcher()", () => { input: { method: "GET", headers: {}, + readHeaders: [], query: { "a": 'b', } @@ -79,6 +89,7 @@ describe("makeSimpleProxyFetcher()", () => { input: { method: "GET", query: {}, + readHeaders: [], headers: {}, }, outputUrl: `https://example.com/proxy?destination=${encodeURIComponent('https://google.com/')}`, @@ -97,6 +108,7 @@ describe("makeSimpleProxyFetcher()", () => { input: { method: "POST", query: {}, + readHeaders: [], headers: {}, }, outputUrl: `https://example.com/proxy?destination=${encodeURIComponent('https://google.com/')}`, @@ -112,6 +124,7 @@ describe("makeSimpleProxyFetcher()", () => { input: { method: "POST", query: {}, + readHeaders: [], headers: {}, }, outputUrl: `https://example.com/proxy?destination=${encodeURIComponent('https://google.com/')}`, diff --git a/src/__test__/fetchers/standard.test.ts b/src/__test__/fetchers/standard.test.ts index 6bad99a..6753500 100644 --- a/src/__test__/fetchers/standard.test.ts +++ b/src/__test__/fetchers/standard.test.ts @@ -16,6 +16,8 @@ describe("makeStandardFetcher()", () => { headers: new Headers({ "content-type": "text/plain", }), + status: 204, + url: "test123", text() { return Promise.resolve(value); }, @@ -24,6 +26,8 @@ describe("makeStandardFetcher()", () => { headers: new Headers({ "content-type": "application/json", }), + status: 204, + url: "test123", json() { return Promise.resolve(value); }, @@ -31,7 +35,11 @@ describe("makeStandardFetcher()", () => { } function expectFetchCall(ops: { inputUrl: string, input: DefaultedFetcherOptions, outputUrl?: string, output: any, outputBody: any }) { - expect(fetcher(ops.inputUrl, ops.input)).resolves.toEqual(ops.outputBody); + const prom = fetcher(ops.inputUrl, ops.input); + expect((async () => (await prom).body)()).resolves.toEqual(ops.outputBody); + expect((async () => (await prom).headers.entries())()).resolves.toEqual((new Headers()).entries()); + expect((async () => (await prom).statusCode)()).resolves.toEqual(204); + expect((async () => (await prom).finalUrl)()).resolves.toEqual("test123"); expect(fetch).toBeCalledWith(ops.outputUrl ?? ops.inputUrl, ops.output); vi.clearAllMocks(); } @@ -43,6 +51,7 @@ describe("makeStandardFetcher()", () => { input: { method: "GET", query: {}, + readHeaders: [], headers: { "X-Hello": "world", }, @@ -53,6 +62,7 @@ describe("makeStandardFetcher()", () => { headers: { "X-Hello": "world", }, + body: undefined, }, outputBody: "hello world" }) @@ -62,6 +72,7 @@ describe("makeStandardFetcher()", () => { input: { method: "GET", headers: {}, + readHeaders: [], query: { "a": 'b', } @@ -79,6 +90,7 @@ describe("makeStandardFetcher()", () => { input: { query: {}, headers: {}, + readHeaders: [], method: "GET" }, outputUrl: "https://google.com/", @@ -97,6 +109,7 @@ describe("makeStandardFetcher()", () => { input: { query: {}, headers: {}, + readHeaders: [], method: "POST" }, outputUrl: "https://google.com/", @@ -112,6 +125,7 @@ describe("makeStandardFetcher()", () => { input: { query: {}, headers: {}, + readHeaders: [], method: "POST" }, outputUrl: "https://google.com/", diff --git a/src/fetchers/standardFetch.ts b/src/fetchers/standardFetch.ts index a17dd92..9fb6afa 100644 --- a/src/fetchers/standardFetch.ts +++ b/src/fetchers/standardFetch.ts @@ -32,7 +32,7 @@ export function makeStandardFetcher(f: FetchLike): Fetcher { let body: any; const isJson = res.headers.get('content-type')?.includes('application/json'); if (isJson) body = await res.json(); - else body = res.text(); + else body = await res.text(); return { body, From 4b1e8288b81d08d8e2bc22191e8cad2f87076722 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 26 Dec 2023 16:14:10 +0100 Subject: [PATCH 11/17] Add headers and preferred headers to streams --- src/providers/streams.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/providers/streams.ts b/src/providers/streams.ts index 03ae873..d56d1a4 100644 --- a/src/providers/streams.ts +++ b/src/providers/streams.ts @@ -9,20 +9,22 @@ export type StreamFile = { export type Qualities = 'unknown' | '360' | '480' | '720' | '1080' | '4k'; -export type FileBasedStream = { - type: 'file'; +type StreamCommon = { id: string; // only unique per output flags: Flags[]; - qualities: Partial>; captions: Caption[]; + headers?: Record; // these headers HAVE to be set to watch the stream + preferredHeaders?: Record; // these headers are optional, would improve the stream }; -export type HlsBasedStream = { +export type FileBasedStream = StreamCommon & { + type: 'file'; + qualities: Partial>; +}; + +export type HlsBasedStream = StreamCommon & { type: 'hls'; - id: string; // only unique per output - flags: Flags[]; playlist: string; - captions: Caption[]; }; export type Stream = FileBasedStream | HlsBasedStream; From af00bcf7c1c0b578ab12f75441c3dd9c9514fb6b Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 26 Dec 2023 16:24:36 +0100 Subject: [PATCH 12/17] export extra fetcher it is --- src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index 347f8dc..95fc631 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,6 @@ export type { EmbedOutput, SourcererOutput } from '@/providers/base'; export type { Stream, StreamFile, FileBasedStream, HlsBasedStream, Qualities } from '@/providers/streams'; -export type { Fetcher, FetcherOptions } from '@/fetchers/types'; +export type { Fetcher, FetcherOptions, FetcherResponse } from '@/fetchers/types'; export type { RunOutput } from '@/runners/runner'; export type { MetaOutput } from '@/entrypoint/utils/meta'; export type { FullScraperEvents } from '@/entrypoint/utils/events'; From feddf9c215791d2e2784357884e7495eb4e9f9de Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 26 Dec 2023 17:00:35 +0100 Subject: [PATCH 13/17] Refactored dev-cli --- package.json | 2 +- src/dev-cli.ts | 423 ---------------------------------------- src/dev-cli/config.ts | 12 ++ src/dev-cli/index.ts | 227 +++++++++++++++++++++ src/dev-cli/logging.ts | 5 + src/dev-cli/tmdb.ts | 95 +++++++++ src/dev-cli/validate.ts | 90 +++++++++ src/providers/base.ts | 39 +++- 8 files changed, 463 insertions(+), 430 deletions(-) delete mode 100644 src/dev-cli.ts create mode 100644 src/dev-cli/config.ts create mode 100644 src/dev-cli/index.ts create mode 100644 src/dev-cli/logging.ts create mode 100644 src/dev-cli/tmdb.ts create mode 100644 src/dev-cli/validate.ts diff --git a/package.json b/package.json index e93259a..7dbd0b3 100644 --- a/package.json +++ b/package.json @@ -35,8 +35,8 @@ "homepage": "https://providers.docs.movie-web.app/", "scripts": { "build": "vite build && tsc --noEmit", + "cli": "ts-node ./src/dev-cli/index.ts", "test": "vitest run", - "test:dev": "ts-node ./src/dev-cli.ts", "test:watch": "vitest", "test:integration": "node ./tests/cjs && node ./tests/esm && node ./tests/browser", "test:coverage": "vitest run --coverage", diff --git a/src/dev-cli.ts b/src/dev-cli.ts deleted file mode 100644 index 54d663a..0000000 --- a/src/dev-cli.ts +++ /dev/null @@ -1,423 +0,0 @@ -/* eslint import/no-extraneous-dependencies: ["error", {"devDependencies": true}] */ - -import util from 'node:util'; - -import { program } from 'commander'; -import dotenv from 'dotenv'; -import { prompt } from 'enquirer'; -import nodeFetch from 'node-fetch'; -import Spinnies from 'spinnies'; - -import { MetaOutput, MovieMedia, ProviderControls, ShowMedia, makeProviders, makeStandardFetcher, targets } from '.'; - -dotenv.config(); - -type ProviderSourceAnswers = { - id: string; - type: string; -}; - -type EmbedSourceAnswers = { - url: string; -}; - -type CommonAnswers = { - fetcher: string; - source: string; -}; - -type ShowAnswers = { - season: string; - episode: string; -}; - -type CommandLineArguments = { - fetcher: string; - sourceId: string; - tmdbId: string; - type: string; - season: string; - episode: string; - url: string; -}; - -const TMDB_API_KEY = process.env.MOVIE_WEB_TMDB_API_KEY ?? ''; - -if (!TMDB_API_KEY?.trim()) { - throw new Error('Missing MOVIE_WEB_TMDB_API_KEY environment variable'); -} - -function logDeepObject(object: Record) { - console.log(util.inspect(object, { showHidden: false, depth: null, colors: true })); -} - -function getAllSources() { - // * The only way to get a list of all sources is to - // * create all these things. Maybe this should change - const providers = makeProviders({ - fetcher: makeStandardFetcher(nodeFetch), - target: targets.NATIVE, - }); - - const combined = [...providers.listSources(), ...providers.listEmbeds()]; - - // * Remove dupes - const map = new Map(combined.map((source) => [source.id, source])); - - return [...map.values()]; -} - -// * Defined here cuz ESLint didn't like the order these were defined in -const sources = getAllSources(); - -async function makeTMDBRequest(url: string): Promise { - const headers: { - accept: 'application/json'; - authorization?: string; - } = { - accept: 'application/json', - }; - - // * Used to get around ESLint - // * Assignment to function parameter 'url'. eslint (no-param-reassign) - let requestURL = url; - - // * JWT keys always start with ey and are ONLY valid as a header. - // * All other keys are ONLY valid as a query param. - // * Thanks TMDB. - if (TMDB_API_KEY.startsWith('ey')) { - headers.authorization = `Bearer ${TMDB_API_KEY}`; - } else { - requestURL += `?api_key=${TMDB_API_KEY}`; - } - - return fetch(requestURL, { - method: 'GET', - headers, - }); -} - -async function getMovieMediaDetails(id: string): Promise { - const response = await makeTMDBRequest(`https://api.themoviedb.org/3/movie/${id}`); - const movie = await response.json(); - - if (movie.success === false) { - throw new Error(movie.status_message); - } - - if (!movie.release_date) { - throw new Error(`${movie.title} has no release_date. Assuming unreleased`); - } - - return { - type: 'movie', - title: movie.title, - releaseYear: Number(movie.release_date.split('-')[0]), - tmdbId: id, - }; -} - -async function getShowMediaDetails(id: string, seasonNumber: string, episodeNumber: string): Promise { - // * TV shows require the TMDB ID for the series, season, and episode - // * and the name of the series. Needs multiple requests - let response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}`); - const series = await response.json(); - - if (series.success === false) { - throw new Error(series.status_message); - } - - if (!series.first_air_date) { - throw new Error(`${series.name} has no first_air_date. Assuming unaired`); - } - - response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}`); - const season = await response.json(); - - if (season.success === false) { - throw new Error(season.status_message); - } - - response = await makeTMDBRequest( - `https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}/episode/${episodeNumber}`, - ); - const episode = await response.json(); - - if (episode.success === false) { - throw new Error(episode.status_message); - } - - return { - type: 'show', - title: series.name, - releaseYear: Number(series.first_air_date.split('-')[0]), - tmdbId: id, - episode: { - number: episode.episode_number, - tmdbId: episode.id, - }, - season: { - number: season.season_number, - tmdbId: season.id, - }, - }; -} - -function joinMediaTypes(mediaTypes: string[] | undefined) { - if (mediaTypes) { - const formatted = mediaTypes - .map((type: string) => { - return `${type[0].toUpperCase() + type.substring(1).toLowerCase()}s`; - }) - .join(' / '); - - return `(${formatted})`; - } - return ''; // * Embed sources pass through here too -} - -async function runScraper(providers: ProviderControls, source: MetaOutput, options: CommandLineArguments) { - const spinnies = new Spinnies(); - - if (source.type === 'embed') { - spinnies.add('scrape', { text: `Running ${source.name} scraper on ${options.url}` }); - try { - const result = await providers.runEmbedScraper({ - url: options.url, - id: source.id, - }); - spinnies.succeed('scrape', { text: 'Done!' }); - logDeepObject(result); - } catch (error) { - let message = 'Unknown error'; - if (error instanceof Error) { - message = error.message; - } - - spinnies.fail('scrape', { text: `ERROR: ${message}` }); - } - } else { - let media; - - if (options.type === 'movie') { - media = await getMovieMediaDetails(options.tmdbId); - } else { - media = await getShowMediaDetails(options.tmdbId, options.season, options.episode); - } - - spinnies.add('scrape', { text: `Running ${source.name} scraper on ${media.title}` }); - try { - const result = await providers.runSourceScraper({ - media, - id: source.id, - }); - spinnies.succeed('scrape', { text: 'Done!' }); - logDeepObject(result); - } catch (error) { - let message = 'Unknown error'; - if (error instanceof Error) { - message = error.message; - } - - spinnies.fail('scrape', { text: `ERROR: ${message}` }); - } - } -} - -async function processOptions(options: CommandLineArguments) { - if (options.fetcher !== 'node-fetch' && options.fetcher !== 'native') { - throw new Error("Fetcher must be either 'native' or 'node-fetch'"); - } - - if (!options.sourceId.trim()) { - throw new Error('Source ID must be provided'); - } - - const source = sources.find(({ id }) => id === options.sourceId); - - if (!source) { - throw new Error('Invalid source ID. No source found'); - } - - if (source.type === 'embed' && !options.url.trim()) { - throw new Error('Must provide an embed URL for embed sources'); - } - - if (source.type === 'source') { - if (!options.tmdbId.trim()) { - throw new Error('Must provide a TMDB ID for provider sources'); - } - - if (Number.isNaN(Number(options.tmdbId)) || Number(options.tmdbId) < 0) { - throw new Error('TMDB ID must be a number greater than 0'); - } - - if (!options.type.trim()) { - throw new Error('Must provide a type for provider sources'); - } - - if (options.type !== 'movie' && options.type !== 'show') { - throw new Error("Invalid media type. Must be either 'movie' or 'show'"); - } - - if (options.type === 'show') { - if (!options.season.trim()) { - throw new Error('Must provide a season number for TV shows'); - } - - if (!options.episode.trim()) { - throw new Error('Must provide an episode number for TV shows'); - } - - if (Number.isNaN(Number(options.season)) || Number(options.season) <= 0) { - throw new Error('Season number must be a number greater than 0'); - } - - if (Number.isNaN(Number(options.episode)) || Number(options.episode) <= 0) { - throw new Error('Episode number must be a number greater than 0'); - } - } - } - - let fetcher; - - if (options.fetcher === 'native') { - fetcher = makeStandardFetcher(fetch); - } else { - fetcher = makeStandardFetcher(nodeFetch); - } - - const providers = makeProviders({ - fetcher, - target: targets.NATIVE, - }); - - await runScraper(providers, source, options); -} - -async function runQuestions() { - const options = { - fetcher: 'node-fetch', - sourceId: '', - tmdbId: '', - type: 'movie', - season: '0', - episode: '0', - url: '', - }; - - const answers = await prompt([ - { - type: 'select', - name: 'fetcher', - message: 'Select a fetcher', - choices: [ - { - message: 'Native', - name: 'native', - }, - { - message: 'Node fetch', - name: 'node-fetch', - }, - ], - }, - { - type: 'select', - name: 'source', - message: 'Select a source', - choices: sources.map((source) => ({ - message: `[${source.type.toLocaleUpperCase()}] ${source.name} ${joinMediaTypes(source.mediaTypes)}`.trim(), - name: source.id, - })), - }, - ]); - - options.fetcher = answers.fetcher; - options.sourceId = answers.source; - - const source = sources.find(({ id }) => id === answers.source); - - if (!source) { - throw new Error(`No source with ID ${answers.source} found`); - } - - if (source.type === 'embed') { - const sourceAnswers = await prompt([ - { - type: 'input', - name: 'url', - message: 'Embed URL', - }, - ]); - - options.url = sourceAnswers.url; - } else { - const sourceAnswers = await prompt([ - { - type: 'input', - name: 'id', - message: 'TMDB ID', - }, - { - type: 'select', - name: 'type', - message: 'Media type', - choices: [ - { - message: 'Movie', - name: 'movie', - }, - { - message: 'TV Show', - name: 'show', - }, - ], - }, - ]); - - options.tmdbId = sourceAnswers.id; - options.type = sourceAnswers.type; - - if (sourceAnswers.type === 'show') { - const seriesAnswers = await prompt([ - { - type: 'input', - name: 'season', - message: 'Season', - }, - { - type: 'input', - name: 'episode', - message: 'Episode', - }, - ]); - - options.season = seriesAnswers.season; - options.episode = seriesAnswers.episode; - } - } - - await processOptions(options); -} - -async function runCommandLine() { - program - .option('-f, --fetcher ', "Fetcher to use. Either 'native' or 'node-fetch'", 'node-fetch') - .option('-sid, --source-id ', 'ID for the source to use. Either an embed or provider', '') - .option('-tid, --tmdb-id ', 'TMDB ID for the media to scrape. Only used if source is a provider', '') - .option('-t, --type ', "Media type. Either 'movie' or 'show'. Only used if source is a provider", 'movie') - .option('-s, --season ', "Season number. Only used if type is 'show'", '0') - .option('-e, --episode ', "Episode number. Only used if type is 'show'", '0') - .option('-u, --url ', 'URL to a video embed. Only used if source is an embed', ''); - - program.parse(); - - await processOptions(program.opts()); -} - -if (process.argv.length === 2) { - runQuestions(); -} else { - runCommandLine(); -} diff --git a/src/dev-cli/config.ts b/src/dev-cli/config.ts new file mode 100644 index 0000000..6deb539 --- /dev/null +++ b/src/dev-cli/config.ts @@ -0,0 +1,12 @@ +export function getConfig() { + let tmdbApiKey = process.env.MOVIE_WEB_TMDB_API_KEY ?? ''; + tmdbApiKey = tmdbApiKey.trim(); + + if (!tmdbApiKey) { + throw new Error('Missing MOVIE_WEB_TMDB_API_KEY environment variable'); + } + + return { + tmdbApiKey, + }; +} diff --git a/src/dev-cli/index.ts b/src/dev-cli/index.ts new file mode 100644 index 0000000..13a8210 --- /dev/null +++ b/src/dev-cli/index.ts @@ -0,0 +1,227 @@ +/* eslint import/no-extraneous-dependencies: ["error", {"devDependencies": true}] */ + +import { program } from 'commander'; +import dotenv from 'dotenv'; +import { prompt } from 'enquirer'; +import Spinnies from 'spinnies'; + +import { logDeepObject } from '@/dev-cli/logging'; +import { getMovieMediaDetails, getShowMediaDetails } from '@/dev-cli/tmdb'; +import { CommandLineArguments, processOptions } from '@/dev-cli/validate'; + +import { MetaOutput, ProviderControls, getBuiltinEmbeds, getBuiltinSources } from '..'; + +dotenv.config(); + +type ProviderSourceAnswers = { + id: string; + type: string; +}; + +type EmbedSourceAnswers = { + url: string; +}; + +type CommonAnswers = { + fetcher: string; + source: string; +}; + +type ShowAnswers = { + season: string; + episode: string; +}; + +const sourceScrapers = getBuiltinSources().sort((a, b) => b.rank - a.rank); +const embedScrapers = getBuiltinEmbeds().sort((a, b) => b.rank - a.rank); +const sources = [...sourceScrapers, ...embedScrapers]; + +function joinMediaTypes(mediaTypes: string[] | undefined) { + if (mediaTypes) { + const formatted = mediaTypes + .map((type: string) => { + return `${type[0].toUpperCase() + type.substring(1).toLowerCase()}s`; + }) + .join(' / '); + + return `(${formatted})`; + } + return ''; // * Embed sources pass through here too +} + +async function runScraper(providers: ProviderControls, source: MetaOutput, options: CommandLineArguments) { + const spinnies = new Spinnies(); + + if (source.type === 'embed') { + spinnies.add('scrape', { text: `Running ${source.name} scraper on ${options.url}` }); + try { + const result = await providers.runEmbedScraper({ + url: options.url, + id: source.id, + }); + spinnies.succeed('scrape', { text: 'Done!' }); + logDeepObject(result); + } catch (error) { + let message = 'Unknown error'; + if (error instanceof Error) { + message = error.message; + } + + spinnies.fail('scrape', { text: `ERROR: ${message}` }); + } + } else { + let media; + + if (options.type === 'movie') { + media = await getMovieMediaDetails(options.tmdbId); + } else { + media = await getShowMediaDetails(options.tmdbId, options.season, options.episode); + } + + spinnies.add('scrape', { text: `Running ${source.name} scraper on ${media.title}` }); + try { + const result = await providers.runSourceScraper({ + media, + id: source.id, + }); + spinnies.succeed('scrape', { text: 'Done!' }); + logDeepObject(result); + } catch (error) { + let message = 'Unknown error'; + if (error instanceof Error) { + message = error.message; + } + + spinnies.fail('scrape', { text: `ERROR: ${message}` }); + } + } +} + +async function runQuestions() { + const options = { + fetcher: 'node-fetch', + sourceId: '', + tmdbId: '', + type: 'movie', + season: '0', + episode: '0', + url: '', + }; + + const answers = await prompt([ + { + type: 'select', + name: 'fetcher', + message: 'Select a fetcher', + choices: [ + { + message: 'Native', + name: 'native', + }, + { + message: 'Node fetch', + name: 'node-fetch', + }, + ], + }, + { + type: 'select', + name: 'source', + message: 'Select a source', + choices: sources.map((source) => ({ + message: `[${source.type.toLocaleUpperCase()}] ${source.name} ${joinMediaTypes(source.mediaTypes)}`.trim(), + name: source.id, + })), + }, + ]); + + options.fetcher = answers.fetcher; + options.sourceId = answers.source; + + const source = sources.find(({ id }) => id === answers.source); + + if (!source) { + throw new Error(`No source with ID ${answers.source} found`); + } + + if (source.type === 'embed') { + const sourceAnswers = await prompt([ + { + type: 'input', + name: 'url', + message: 'Embed URL', + }, + ]); + + options.url = sourceAnswers.url; + } else { + const sourceAnswers = await prompt([ + { + type: 'input', + name: 'id', + message: 'TMDB ID', + }, + { + type: 'select', + name: 'type', + message: 'Media type', + choices: [ + { + message: 'Movie', + name: 'movie', + }, + { + message: 'TV Show', + name: 'show', + }, + ], + }, + ]); + + options.tmdbId = sourceAnswers.id; + options.type = sourceAnswers.type; + + if (sourceAnswers.type === 'show') { + const seriesAnswers = await prompt([ + { + type: 'input', + name: 'season', + message: 'Season', + }, + { + type: 'input', + name: 'episode', + message: 'Episode', + }, + ]); + + options.season = seriesAnswers.season; + options.episode = seriesAnswers.episode; + } + } + + const { providers, source: validatedSource, options: validatedOps } = await processOptions(sources, options); + await runScraper(providers, validatedSource, validatedOps); +} + +async function runCommandLine() { + program + .option('-f, --fetcher ', "Fetcher to use. Either 'native' or 'node-fetch'", 'node-fetch') + .option('-sid, --source-id ', 'ID for the source to use. Either an embed or provider', '') + .option('-tid, --tmdb-id ', 'TMDB ID for the media to scrape. Only used if source is a provider', '') + .option('-t, --type ', "Media type. Either 'movie' or 'show'. Only used if source is a provider", 'movie') + .option('-s, --season ', "Season number. Only used if type is 'show'", '0') + .option('-e, --episode ', "Episode number. Only used if type is 'show'", '0') + .option('-u, --url ', 'URL to a video embed. Only used if source is an embed', ''); + + program.parse(); + + const { providers, source: validatedSource, options: validatedOps } = await processOptions(sources, program.opts()); + await runScraper(providers, validatedSource, validatedOps); +} + +if (process.argv.length === 2) { + runQuestions().catch(() => console.error('Exited.')); +} else { + runCommandLine().catch(() => console.error('Exited.')); +} diff --git a/src/dev-cli/logging.ts b/src/dev-cli/logging.ts new file mode 100644 index 0000000..1a519f4 --- /dev/null +++ b/src/dev-cli/logging.ts @@ -0,0 +1,5 @@ +import { inspect } from 'node:util'; + +export function logDeepObject(object: Record) { + console.log(inspect(object, { showHidden: false, depth: null, colors: true })); +} diff --git a/src/dev-cli/tmdb.ts b/src/dev-cli/tmdb.ts new file mode 100644 index 0000000..7490336 --- /dev/null +++ b/src/dev-cli/tmdb.ts @@ -0,0 +1,95 @@ +import { getConfig } from '@/dev-cli/config'; + +import { MovieMedia, ShowMedia } from '..'; + +export async function makeTMDBRequest(url: string): Promise { + const headers: { + accept: 'application/json'; + authorization?: string; + } = { + accept: 'application/json', + }; + + let requestURL = url; + const key = getConfig().tmdbApiKey; + + // * JWT keys always start with ey and are ONLY valid as a header. + // * All other keys are ONLY valid as a query param. + // * Thanks TMDB. + if (key.startsWith('ey')) { + headers.authorization = `Bearer ${key}`; + } else { + requestURL += `?api_key=${key}`; + } + + return fetch(requestURL, { + method: 'GET', + headers, + }); +} + +export async function getMovieMediaDetails(id: string): Promise { + const response = await makeTMDBRequest(`https://api.themoviedb.org/3/movie/${id}`); + const movie = await response.json(); + + if (movie.success === false) { + throw new Error(movie.status_message); + } + + if (!movie.release_date) { + throw new Error(`${movie.title} has no release_date. Assuming unreleased`); + } + + return { + type: 'movie', + title: movie.title, + releaseYear: Number(movie.release_date.split('-')[0]), + tmdbId: id, + }; +} + +export async function getShowMediaDetails(id: string, seasonNumber: string, episodeNumber: string): Promise { + // * TV shows require the TMDB ID for the series, season, and episode + // * and the name of the series. Needs multiple requests + let response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}`); + const series = await response.json(); + + if (series.success === false) { + throw new Error(series.status_message); + } + + if (!series.first_air_date) { + throw new Error(`${series.name} has no first_air_date. Assuming unaired`); + } + + response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}`); + const season = await response.json(); + + if (season.success === false) { + throw new Error(season.status_message); + } + + response = await makeTMDBRequest( + `https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}/episode/${episodeNumber}`, + ); + const episode = await response.json(); + + if (episode.success === false) { + throw new Error(episode.status_message); + } + + return { + type: 'show', + title: series.name, + releaseYear: Number(series.first_air_date.split('-')[0]), + tmdbId: id, + episode: { + number: episode.episode_number, + tmdbId: episode.id, + }, + season: { + number: season.season_number, + tmdbId: season.id, + }, + }; +} diff --git a/src/dev-cli/validate.ts b/src/dev-cli/validate.ts new file mode 100644 index 0000000..60c8340 --- /dev/null +++ b/src/dev-cli/validate.ts @@ -0,0 +1,90 @@ +import nodeFetch from 'node-fetch'; + +import { Embed, Sourcerer } from '@/providers/base'; + +import { makeProviders, makeStandardFetcher, targets } from '..'; + +export type CommandLineArguments = { + fetcher: string; + sourceId: string; + tmdbId: string; + type: string; + season: string; + episode: string; + url: string; +}; + +export async function processOptions(sources: Array, options: CommandLineArguments) { + if (options.fetcher !== 'node-fetch' && options.fetcher !== 'native') { + throw new Error("Fetcher must be either 'native' or 'node-fetch'"); + } + + if (!options.sourceId.trim()) { + throw new Error('Source ID must be provided'); + } + + const source = sources.find(({ id }) => id === options.sourceId); + + if (!source) { + throw new Error('Invalid source ID. No source found'); + } + + if (source.type === 'embed' && !options.url.trim()) { + throw new Error('Must provide an embed URL for embed sources'); + } + + if (source.type === 'source') { + if (!options.tmdbId.trim()) { + throw new Error('Must provide a TMDB ID for provider sources'); + } + + if (Number.isNaN(Number(options.tmdbId)) || Number(options.tmdbId) < 0) { + throw new Error('TMDB ID must be a number greater than 0'); + } + + if (!options.type.trim()) { + throw new Error('Must provide a type for provider sources'); + } + + if (options.type !== 'movie' && options.type !== 'show') { + throw new Error("Invalid media type. Must be either 'movie' or 'show'"); + } + + if (options.type === 'show') { + if (!options.season.trim()) { + throw new Error('Must provide a season number for TV shows'); + } + + if (!options.episode.trim()) { + throw new Error('Must provide an episode number for TV shows'); + } + + if (Number.isNaN(Number(options.season)) || Number(options.season) <= 0) { + throw new Error('Season number must be a number greater than 0'); + } + + if (Number.isNaN(Number(options.episode)) || Number(options.episode) <= 0) { + throw new Error('Episode number must be a number greater than 0'); + } + } + } + + let fetcher; + + if (options.fetcher === 'native') { + fetcher = makeStandardFetcher(fetch); + } else { + fetcher = makeStandardFetcher(nodeFetch); + } + + const providers = makeProviders({ + fetcher, + target: targets.NATIVE, + }); + + return { + providers, + options, + source, + }; +} diff --git a/src/providers/base.ts b/src/providers/base.ts index 7371166..0d43895 100644 --- a/src/providers/base.ts +++ b/src/providers/base.ts @@ -2,6 +2,8 @@ import { Flags } from '@/entrypoint/utils/targets'; import { Stream } from '@/providers/streams'; import { EmbedScrapeContext, MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; +export type MediaScraperTypes = 'show' | 'movie'; + export type SourcererEmbed = { embedId: string; url: string; @@ -12,7 +14,7 @@ export type SourcererOutput = { stream?: Stream[]; }; -export type Sourcerer = { +export type SourcererOptions = { id: string; name: string; // displayed in the UI rank: number; // the higher the number, the earlier it gets put on the queue @@ -22,15 +24,29 @@ export type Sourcerer = { scrapeShow?: (input: ShowScrapeContext) => Promise; }; -export function makeSourcerer(state: Sourcerer): Sourcerer { - return state; +export type Sourcerer = SourcererOptions & { + type: 'source'; + disabled: boolean; + mediaTypes: MediaScraperTypes[]; +}; + +export function makeSourcerer(state: SourcererOptions): Sourcerer { + const mediaTypes: MediaScraperTypes[] = []; + if (state.scrapeMovie) mediaTypes.push('movie'); + if (state.scrapeShow) mediaTypes.push('show'); + return { + ...state, + type: 'source', + disabled: state.disabled ?? false, + mediaTypes, + }; } export type EmbedOutput = { stream: Stream[]; }; -export type Embed = { +export type EmbedOptions = { id: string; name: string; // displayed in the UI rank: number; // the higher the number, the earlier it gets put on the queue @@ -38,6 +54,17 @@ export type Embed = { scrape: (input: EmbedScrapeContext) => Promise; }; -export function makeEmbed(state: Embed): Embed { - return state; +export type Embed = EmbedOptions & { + type: 'embed'; + disabled: boolean; + mediaTypes: undefined; +}; + +export function makeEmbed(state: EmbedOptions): Embed { + return { + ...state, + type: 'embed', + disabled: state.disabled ?? false, + mediaTypes: undefined, + }; } From aa4b7cda9e837213b2f9b57447697fcbb85a1f57 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 26 Dec 2023 17:01:02 +0100 Subject: [PATCH 14/17] Update readme with new cli command --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d30d021..8b7633c 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ features: Visit documentation here: https://providers.docs.movie-web.app/ ## Development -To make testing scrapers easier during development a CLI tool is available to run specific sources. To run the CLI testing tool, use `npm run test:dev`. The script supports 2 execution modes +To make testing scrapers easier during development a CLI tool is available to run specific sources. To run the CLI testing tool, use `npm run cli`. The script supports 2 execution modes - CLI Mode, for passing in arguments directly to the script - Question Mode, where the script asks you questions about which source you wish to test @@ -31,5 +31,5 @@ The following CLI Mode arguments are available Example testing the FlixHQ source on the movie "Spirited Away" ```bash -npm run test:dev -- -sid flixhq -tid 129 -t movie +npm run cli -- -sid flixhq -tid 129 -t movie ``` From edd08446cf3ffc39acf6aa698c5996a1201c0d27 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 26 Dec 2023 17:22:23 +0100 Subject: [PATCH 15/17] Move runScraper to use the options instead of providers itself --- src/dev-cli/index.ts | 17 +++++++++++------ src/dev-cli/validate.ts | 15 ++++++++------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/dev-cli/index.ts b/src/dev-cli/index.ts index 13a8210..52f1236 100644 --- a/src/dev-cli/index.ts +++ b/src/dev-cli/index.ts @@ -9,7 +9,7 @@ import { logDeepObject } from '@/dev-cli/logging'; import { getMovieMediaDetails, getShowMediaDetails } from '@/dev-cli/tmdb'; import { CommandLineArguments, processOptions } from '@/dev-cli/validate'; -import { MetaOutput, ProviderControls, getBuiltinEmbeds, getBuiltinSources } from '..'; +import { MetaOutput, ProviderMakerOptions, getBuiltinEmbeds, getBuiltinSources, makeProviders } from '..'; dotenv.config(); @@ -49,8 +49,9 @@ function joinMediaTypes(mediaTypes: string[] | undefined) { return ''; // * Embed sources pass through here too } -async function runScraper(providers: ProviderControls, source: MetaOutput, options: CommandLineArguments) { +async function runScraper(providerOptions: ProviderMakerOptions, source: MetaOutput, options: CommandLineArguments) { const spinnies = new Spinnies(); + const providers = makeProviders(providerOptions); if (source.type === 'embed') { spinnies.add('scrape', { text: `Running ${source.name} scraper on ${options.url}` }); @@ -200,8 +201,8 @@ async function runQuestions() { } } - const { providers, source: validatedSource, options: validatedOps } = await processOptions(sources, options); - await runScraper(providers, validatedSource, validatedOps); + const { providerOptions, source: validatedSource, options: validatedOps } = await processOptions(sources, options); + await runScraper(providerOptions, validatedSource, validatedOps); } async function runCommandLine() { @@ -216,8 +217,12 @@ async function runCommandLine() { program.parse(); - const { providers, source: validatedSource, options: validatedOps } = await processOptions(sources, program.opts()); - await runScraper(providers, validatedSource, validatedOps); + const { + providerOptions, + source: validatedSource, + options: validatedOps, + } = await processOptions(sources, program.opts()); + await runScraper(providerOptions, validatedSource, validatedOps); } if (process.argv.length === 2) { diff --git a/src/dev-cli/validate.ts b/src/dev-cli/validate.ts index 60c8340..b600454 100644 --- a/src/dev-cli/validate.ts +++ b/src/dev-cli/validate.ts @@ -2,7 +2,7 @@ import nodeFetch from 'node-fetch'; import { Embed, Sourcerer } from '@/providers/base'; -import { makeProviders, makeStandardFetcher, targets } from '..'; +import { ProviderMakerOptions, makeStandardFetcher, targets } from '..'; export type CommandLineArguments = { fetcher: string; @@ -15,8 +15,9 @@ export type CommandLineArguments = { }; export async function processOptions(sources: Array, options: CommandLineArguments) { - if (options.fetcher !== 'node-fetch' && options.fetcher !== 'native') { - throw new Error("Fetcher must be either 'native' or 'node-fetch'"); + const fetcherOptions = ['node-fetch', 'native', 'browser']; + if (!fetcherOptions.includes(options.fetcher)) { + throw new Error(`Fetcher must be any of: ${fetcherOptions.join()}`); } if (!options.sourceId.trim()) { @@ -77,13 +78,13 @@ export async function processOptions(sources: Array, options: fetcher = makeStandardFetcher(nodeFetch); } - const providers = makeProviders({ + const providerOptions: ProviderMakerOptions = { fetcher, - target: targets.NATIVE, - }); + target: targets.ANY, + }; return { - providers, + providerOptions, options, source, }; From 75d4b9edcb8dd291bc3e4dc8abf29d56c33f7cf8 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 26 Dec 2023 17:39:40 +0100 Subject: [PATCH 16/17] Seperate scraper code from index cli file --- src/dev-cli/config.ts | 4 +++ src/dev-cli/index.ts | 57 ++------------------------------------ src/dev-cli/scraper.ts | 63 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 54 deletions(-) create mode 100644 src/dev-cli/scraper.ts diff --git a/src/dev-cli/config.ts b/src/dev-cli/config.ts index 6deb539..38f8039 100644 --- a/src/dev-cli/config.ts +++ b/src/dev-cli/config.ts @@ -6,7 +6,11 @@ export function getConfig() { throw new Error('Missing MOVIE_WEB_TMDB_API_KEY environment variable'); } + let proxyUrl: undefined | string = process.env.MOVIE_WEB_PROXY_URL; + proxyUrl = !proxyUrl ? undefined : proxyUrl; + return { tmdbApiKey, + proxyUrl, }; } diff --git a/src/dev-cli/index.ts b/src/dev-cli/index.ts index 52f1236..1689a37 100644 --- a/src/dev-cli/index.ts +++ b/src/dev-cli/index.ts @@ -3,13 +3,11 @@ import { program } from 'commander'; import dotenv from 'dotenv'; import { prompt } from 'enquirer'; -import Spinnies from 'spinnies'; -import { logDeepObject } from '@/dev-cli/logging'; -import { getMovieMediaDetails, getShowMediaDetails } from '@/dev-cli/tmdb'; -import { CommandLineArguments, processOptions } from '@/dev-cli/validate'; +import { runScraper } from '@/dev-cli/scraper'; +import { processOptions } from '@/dev-cli/validate'; -import { MetaOutput, ProviderMakerOptions, getBuiltinEmbeds, getBuiltinSources, makeProviders } from '..'; +import { getBuiltinEmbeds, getBuiltinSources } from '..'; dotenv.config(); @@ -49,55 +47,6 @@ function joinMediaTypes(mediaTypes: string[] | undefined) { return ''; // * Embed sources pass through here too } -async function runScraper(providerOptions: ProviderMakerOptions, source: MetaOutput, options: CommandLineArguments) { - const spinnies = new Spinnies(); - const providers = makeProviders(providerOptions); - - if (source.type === 'embed') { - spinnies.add('scrape', { text: `Running ${source.name} scraper on ${options.url}` }); - try { - const result = await providers.runEmbedScraper({ - url: options.url, - id: source.id, - }); - spinnies.succeed('scrape', { text: 'Done!' }); - logDeepObject(result); - } catch (error) { - let message = 'Unknown error'; - if (error instanceof Error) { - message = error.message; - } - - spinnies.fail('scrape', { text: `ERROR: ${message}` }); - } - } else { - let media; - - if (options.type === 'movie') { - media = await getMovieMediaDetails(options.tmdbId); - } else { - media = await getShowMediaDetails(options.tmdbId, options.season, options.episode); - } - - spinnies.add('scrape', { text: `Running ${source.name} scraper on ${media.title}` }); - try { - const result = await providers.runSourceScraper({ - media, - id: source.id, - }); - spinnies.succeed('scrape', { text: 'Done!' }); - logDeepObject(result); - } catch (error) { - let message = 'Unknown error'; - if (error instanceof Error) { - message = error.message; - } - - spinnies.fail('scrape', { text: `ERROR: ${message}` }); - } - } -} - async function runQuestions() { const options = { fetcher: 'node-fetch', diff --git a/src/dev-cli/scraper.ts b/src/dev-cli/scraper.ts new file mode 100644 index 0000000..c72e36e --- /dev/null +++ b/src/dev-cli/scraper.ts @@ -0,0 +1,63 @@ +/* eslint import/no-extraneous-dependencies: ["error", {"devDependencies": true}] */ + +import Spinnies from 'spinnies'; + +import { logDeepObject } from '@/dev-cli/logging'; +import { getMovieMediaDetails, getShowMediaDetails } from '@/dev-cli/tmdb'; +import { CommandLineArguments } from '@/dev-cli/validate'; + +import { MetaOutput, ProviderMakerOptions, makeProviders } from '..'; + +async function runActualScraping( + providerOptions: ProviderMakerOptions, + source: MetaOutput, + options: CommandLineArguments, +) { + const providers = makeProviders(providerOptions); + + if (source.type === 'embed') { + return providers.runEmbedScraper({ + url: options.url, + id: source.id, + }); + } + + if (source.type === 'source') { + let media; + + if (options.type === 'movie') { + media = await getMovieMediaDetails(options.tmdbId); + } else { + media = await getShowMediaDetails(options.tmdbId, options.season, options.episode); + } + + return providers.runSourceScraper({ + media, + id: source.id, + }); + } + + throw new Error('Invalid source type'); +} + +export async function runScraper( + providerOptions: ProviderMakerOptions, + source: MetaOutput, + options: CommandLineArguments, +) { + const spinnies = new Spinnies(); + + spinnies.add('scrape', { text: `Running ${source.name} scraper` }); + try { + const result = await runActualScraping(providerOptions, source, options); + spinnies.succeed('scrape', { text: 'Done!' }); + logDeepObject(result); + } catch (error) { + let message = 'Unknown error'; + if (error instanceof Error) { + message = error.message; + } + spinnies.fail('scrape', { text: `ERROR: ${message}` }); + console.error(error); + } +} From c26e135d745297a46b2c35882463205f5551f50a Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 26 Dec 2023 19:13:15 +0100 Subject: [PATCH 17/17] dev-cli with browser based fetching --- src/dev-cli/browser/.gitignore | 1 + src/dev-cli/browser/index.html | 11 +++++ src/dev-cli/browser/index.ts | 17 ++++++++ src/dev-cli/index.ts | 6 ++- src/dev-cli/scraper.ts | 77 +++++++++++++++++++++++++++++++++- 5 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 src/dev-cli/browser/.gitignore create mode 100644 src/dev-cli/browser/index.html create mode 100644 src/dev-cli/browser/index.ts diff --git a/src/dev-cli/browser/.gitignore b/src/dev-cli/browser/.gitignore new file mode 100644 index 0000000..1521c8b --- /dev/null +++ b/src/dev-cli/browser/.gitignore @@ -0,0 +1 @@ +dist diff --git a/src/dev-cli/browser/index.html b/src/dev-cli/browser/index.html new file mode 100644 index 0000000..7709f4b --- /dev/null +++ b/src/dev-cli/browser/index.html @@ -0,0 +1,11 @@ + + + + + + Scraper CLI + + + + + diff --git a/src/dev-cli/browser/index.ts b/src/dev-cli/browser/index.ts new file mode 100644 index 0000000..d1f6494 --- /dev/null +++ b/src/dev-cli/browser/index.ts @@ -0,0 +1,17 @@ +import { makeProviders, makeSimpleProxyFetcher, makeStandardFetcher, targets } from '../../../lib'; + +(window as any).scrape = (proxyUrl: string, type: 'source' | 'embed', input: any) => { + const providers = makeProviders({ + fetcher: makeStandardFetcher(fetch), + target: targets.BROWSER, + proxiedFetcher: makeSimpleProxyFetcher(proxyUrl, fetch), + }); + if (type === 'source') { + return providers.runSourceScraper(input); + } + if (type === 'embed') { + return providers.runEmbedScraper(input); + } + + throw new Error('Input input type'); +}; diff --git a/src/dev-cli/index.ts b/src/dev-cli/index.ts index 1689a37..bd95599 100644 --- a/src/dev-cli/index.ts +++ b/src/dev-cli/index.ts @@ -62,7 +62,7 @@ async function runQuestions() { { type: 'select', name: 'fetcher', - message: 'Select a fetcher', + message: 'Select a fetcher mode', choices: [ { message: 'Native', @@ -72,6 +72,10 @@ async function runQuestions() { message: 'Node fetch', name: 'node-fetch', }, + { + message: 'Browser', + name: 'browser', + }, ], }, { diff --git a/src/dev-cli/scraper.ts b/src/dev-cli/scraper.ts index c72e36e..882d321 100644 --- a/src/dev-cli/scraper.ts +++ b/src/dev-cli/scraper.ts @@ -1,18 +1,91 @@ /* eslint import/no-extraneous-dependencies: ["error", {"devDependencies": true}] */ -import Spinnies from 'spinnies'; +import { existsSync } from 'fs'; +import { join } from 'path'; +import puppeteer, { Browser } from 'puppeteer'; +import Spinnies from 'spinnies'; +import { PreviewServer, build, preview } from 'vite'; + +import { getConfig } from '@/dev-cli/config'; import { logDeepObject } from '@/dev-cli/logging'; import { getMovieMediaDetails, getShowMediaDetails } from '@/dev-cli/tmdb'; import { CommandLineArguments } from '@/dev-cli/validate'; import { MetaOutput, ProviderMakerOptions, makeProviders } from '..'; -async function runActualScraping( +async function runBrowserScraping( providerOptions: ProviderMakerOptions, source: MetaOutput, options: CommandLineArguments, ) { + if (!existsSync(join(__dirname, '../../lib/index.mjs'))) + throw new Error('Please compile before running cli in browser mode'); + const config = getConfig(); + if (!config.proxyUrl) + throw new Error('Simple proxy url must be set in the environment (MOVIE_WEB_PROXY_URL) for browser mode to work'); + + const root = join(__dirname, 'browser'); + let server: PreviewServer | undefined; + let browser: Browser | undefined; + try { + // setup browser + await build({ + root, + }); + server = await preview({ + root, + }); + browser = await puppeteer.launch({ + headless: 'new', + args: ['--no-sandbox', '--disable-setuid-sandbox'], + }); + const page = await browser.newPage(); + await page.goto(server.resolvedUrls.local[0]); + await page.waitForFunction('!!window.scrape', { timeout: 5000 }); + + // get input media + let input: any; + if (source.type === 'embed') { + input = { + url: options.url, + id: source.id, + }; + } else if (source.type === 'source') { + let media; + if (options.type === 'movie') { + media = await getMovieMediaDetails(options.tmdbId); + } else { + media = await getShowMediaDetails(options.tmdbId, options.season, options.episode); + } + input = { + media, + id: source.id, + }; + } else { + throw new Error('Wrong source input type'); + } + + return await page.evaluate( + async (proxy, type, inp) => { + return (window as any).scrape(proxy, type, inp); + }, + config.proxyUrl, + source.type, + input, + ); + } finally { + server?.httpServer.close(); + await browser?.close(); + } +} + +async function runActualScraping( + providerOptions: ProviderMakerOptions, + source: MetaOutput, + options: CommandLineArguments, +): Promise { + if (options.fetcher === 'browser') return runBrowserScraping(providerOptions, source, options); const providers = makeProviders(providerOptions); if (source.type === 'embed') {