From 213f8f56ab8a48aabb08e7996eb30bbb687925cf Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Mon, 25 Sep 2023 18:50:56 -0400 Subject: [PATCH 01/17] Added source testing script --- .gitignore | 1 + package-lock.json | 385 ++++++++++++++++++++++++++++++------------- package.json | 8 +- run-source.ts | 405 ++++++++++++++++++++++++++++++++++++++++++++++ tsconfig.json | 2 +- 5 files changed, 685 insertions(+), 116 deletions(-) create mode 100644 run-source.ts diff --git a/.gitignore b/.gitignore index 7525992..7780010 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ node_modules/ /lib coverage +.env diff --git a/package-lock.json b/package-lock.json index 626dcc4..26d6d86 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,24 +1,29 @@ { "name": "@movie-web/providers", - "version": "0.0.11", + "version": "0.0.12", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@movie-web/providers", - "version": "0.0.11", + "version": "0.0.12", "license": "MIT", "dependencies": { "cheerio": "^1.0.0-rc.12", "crypto-js": "^4.1.1", "form-data": "^4.0.0", - "node-fetch": "^3.3.2" + "node-fetch": "^2.7.0" }, "devDependencies": { "@types/crypto-js": "^4.1.1", + "@types/node-fetch": "^2.6.6", + "@types/spinnies": "^0.5.1", "@typescript-eslint/eslint-plugin": "^5.60.0", "@typescript-eslint/parser": "^5.60.0", "@vitest/coverage-v8": "^0.34.3", + "commander": "^11.0.0", + "dotenv": "^16.3.1", + "enquirer": "^2.4.1", "eslint": "^8.30.0", "eslint-config-airbnb-base": "^15.0.0", "eslint-config-prettier": "^8.5.0", @@ -26,6 +31,7 @@ "eslint-plugin-import": "^2.27.5", "eslint-plugin-prettier": "^4.2.1", "prettier": "^2.6.2", + "spinnies": "^0.5.1", "tsc-alias": "^1.6.7", "typescript": "^4.6.3", "vite": "^4.0.0", @@ -877,12 +883,28 @@ "integrity": "sha512-EhcH/wvidPy1WeML3TtYFGR83UzjxeWRen9V402T8aUGYsCHOmfoisV3ZSg03gAFIbLq8TnWOJ0f4cALtnSEUg==", "dev": true }, + "node_modules/@types/node-fetch": { + "version": "2.6.6", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.6.tgz", + "integrity": "sha512-95X8guJYhfqiuVVhRFxVQcf4hW/2bCuoPwDasMf/531STFoNoWTT7YDnWdXHEZKqAGUigmpG31r2FE70LwnzJw==", + "dev": true, + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.0" + } + }, "node_modules/@types/semver": { "version": "7.5.0", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.0.tgz", "integrity": "sha512-G8hZ6XJiHnuhQKR7ZmysCeJWE08o8T0AXtk5darsCaTVsYZhhgUrq53jizaR2FvsoeCwJhlmwTjkXBY5Pn/ZHw==", "dev": true }, + "node_modules/@types/spinnies": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/@types/spinnies/-/spinnies-0.5.1.tgz", + "integrity": "sha512-ka85pZ8mW3QPeKxDqrpNMNzoa/oEdlOXXvckCneDcKddj52zM5fHpiAPVQIpYZUMIw66nZsc1CK1dcB0h1a37g==", + "dev": true + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "5.60.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.60.0.tgz", @@ -1555,6 +1577,15 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ansi-colors": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", + "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -1953,6 +1984,18 @@ "node": ">= 6" } }, + "node_modules/cli-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", + "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", + "dev": true, + "dependencies": { + "restore-cursor": "^3.1.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -1992,12 +2035,12 @@ } }, "node_modules/commander": { - "version": "9.5.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz", - "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==", + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-11.0.0.tgz", + "integrity": "sha512-9HMlXtt/BNoYr8ooyjjNRdIilOTkVJXB+GhxMTtOKwk0R4j4lS4NpjuqmRxroBfnfTSHQIHQB7wryHhXarNjmQ==", "dev": true, "engines": { - "node": "^12.20.0 || >=14" + "node": ">=16" } }, "node_modules/concat-map": { @@ -2127,14 +2170,6 @@ "optional": true, "peer": true }, - "node_modules/data-uri-to-buffer": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", - "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", - "engines": { - "node": ">= 12" - } - }, "node_modules/data-urls": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz", @@ -2492,6 +2527,18 @@ "url": "https://github.com/fb55/domutils?sponsor=1" } }, + "node_modules/dotenv": { + "version": "16.3.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", + "integrity": "sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/motdotla/dotenv?sponsor=1" + } + }, "node_modules/enhanced-resolve": { "version": "5.15.0", "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.15.0.tgz", @@ -2505,6 +2552,19 @@ "node": ">=10.13.0" } }, + "node_modules/enquirer": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.4.1.tgz", + "integrity": "sha512-rRqJg/6gd538VHvR3PSrdRBb/1Vy2YfzHqzvbhGIQpDRKIa4FgV/54b5Q1xYSxOOwKvjXweS26E0Q+nAMwp2pQ==", + "dev": true, + "dependencies": { + "ansi-colors": "^4.1.1", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8.6" + } + }, "node_modules/entities": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", @@ -2976,6 +3036,30 @@ "node": ">=0.10.0" } }, + "node_modules/eslint-plugin-import/node_modules/json5": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz", + "integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==", + "dev": true, + "dependencies": { + "minimist": "^1.2.0" + }, + "bin": { + "json5": "lib/cli.js" + } + }, + "node_modules/eslint-plugin-import/node_modules/tsconfig-paths": { + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.14.2.tgz", + "integrity": "sha512-o/9iXgCYc5L/JxCHPe3Hvh8Q/2xm5Z+p18PESBU6Ff33695QnCHBEjcytY2q19ua7Mbl/DavtBOLq+oG0RCL+g==", + "dev": true, + "dependencies": { + "@types/json5": "^0.0.29", + "json5": "^1.0.2", + "minimist": "^1.2.6", + "strip-bom": "^3.0.0" + } + }, "node_modules/eslint-plugin-prettier": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-4.2.1.tgz", @@ -3189,28 +3273,6 @@ "reusify": "^1.0.4" } }, - "node_modules/fetch-blob": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", - "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "paypal", - "url": "https://paypal.me/jimmywarting" - } - ], - "dependencies": { - "node-domexception": "^1.0.0", - "web-streams-polyfill": "^3.0.3" - }, - "engines": { - "node": "^12.20 || >= 14.13" - } - }, "node_modules/file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -3292,17 +3354,6 @@ "node": ">= 6" } }, - "node_modules/formdata-polyfill": { - "version": "4.0.10", - "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", - "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", - "dependencies": { - "fetch-blob": "^3.1.2" - }, - "engines": { - "node": ">=12.20.0" - } - }, "node_modules/fs-extra": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz", @@ -4589,39 +4640,42 @@ "integrity": "sha512-Tj+HTDSJJKaZnfiuw+iaF9skdPpTo2GtEly5JHnWV/hfv2Qj/9RKsGISQtLh2ox3l5EAGw487hnBee0sIJ6v2g==", "dev": true }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, "engines": { - "node": ">=10.5.0" + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } } }, - "node_modules/node-fetch": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "node_modules/node-fetch/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, + "node_modules/node-fetch/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "node_modules/node-fetch/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", "dependencies": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-fetch" + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" } }, "node_modules/normalize-path": { @@ -5177,6 +5231,19 @@ "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" } }, + "node_modules/restore-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", + "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", + "dev": true, + "dependencies": { + "onetime": "^5.1.0", + "signal-exit": "^3.0.2" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/reusify": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", @@ -5375,6 +5442,109 @@ "node": ">=0.10.0" } }, + "node_modules/spinnies": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/spinnies/-/spinnies-0.5.1.tgz", + "integrity": "sha512-WpjSXv9NQz0nU3yCT9TFEOfpFrXADY9C5fG6eAJqixLhvTX1jP3w92Y8IE5oafIe42nlF9otjhllnXN/QCaB3A==", + "dev": true, + "dependencies": { + "chalk": "^2.4.2", + "cli-cursor": "^3.0.0", + "strip-ansi": "^5.2.0" + } + }, + "node_modules/spinnies/node_modules/ansi-regex": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", + "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/spinnies/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/spinnies/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/spinnies/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/spinnies/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true + }, + "node_modules/spinnies/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/spinnies/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/spinnies/node_modules/strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "dependencies": { + "ansi-regex": "^4.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/spinnies/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -5459,6 +5629,15 @@ "node": ">=8" } }, + "node_modules/strip-bom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", + "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==", + "dev": true, + "engines": { + "node": ">=4" + } + }, "node_modules/strip-final-newline": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", @@ -5680,37 +5859,13 @@ "tsc-alias": "dist/bin/index.js" } }, - "node_modules/tsconfig-paths": { - "version": "3.14.2", - "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.14.2.tgz", - "integrity": "sha512-o/9iXgCYc5L/JxCHPe3Hvh8Q/2xm5Z+p18PESBU6Ff33695QnCHBEjcytY2q19ua7Mbl/DavtBOLq+oG0RCL+g==", - "dev": true, - "dependencies": { - "@types/json5": "^0.0.29", - "json5": "^1.0.2", - "minimist": "^1.2.6", - "strip-bom": "^3.0.0" - } - }, - "node_modules/tsconfig-paths/node_modules/json5": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz", - "integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==", - "dev": true, - "dependencies": { - "minimist": "^1.2.0" - }, - "bin": { - "json5": "lib/cli.js" - } - }, - "node_modules/tsconfig-paths/node_modules/strip-bom": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", - "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==", + "node_modules/tsc-alias/node_modules/commander": { + "version": "9.5.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz", + "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==", "dev": true, "engines": { - "node": ">=4" + "node": "^12.20.0 || >=14" } }, "node_modules/tslib": { @@ -6180,14 +6335,6 @@ "node": ">=10" } }, - "node_modules/web-streams-polyfill": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", - "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==", - "engines": { - "node": ">= 8" - } - }, "node_modules/webidl-conversions": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-6.1.0.tgz", @@ -6401,6 +6548,16 @@ "optionalDependencies": { "commander": "^9.4.1" } + }, + "node_modules/z-schema/node_modules/commander": { + "version": "9.5.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz", + "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==", + "dev": true, + "optional": true, + "engines": { + "node": "^12.20.0 || >=14" + } } } } diff --git a/package.json b/package.json index 95b1b79..8ce3f6d 100644 --- a/package.json +++ b/package.json @@ -46,9 +46,14 @@ }, "devDependencies": { "@types/crypto-js": "^4.1.1", + "@types/node-fetch": "^2.6.6", + "@types/spinnies": "^0.5.1", "@typescript-eslint/eslint-plugin": "^5.60.0", "@typescript-eslint/parser": "^5.60.0", "@vitest/coverage-v8": "^0.34.3", + "commander": "^11.0.0", + "dotenv": "^16.3.1", + "enquirer": "^2.4.1", "eslint": "^8.30.0", "eslint-config-airbnb-base": "^15.0.0", "eslint-config-prettier": "^8.5.0", @@ -56,6 +61,7 @@ "eslint-plugin-import": "^2.27.5", "eslint-plugin-prettier": "^4.2.1", "prettier": "^2.6.2", + "spinnies": "^0.5.1", "tsc-alias": "^1.6.7", "typescript": "^4.6.3", "vite": "^4.0.0", @@ -67,6 +73,6 @@ "cheerio": "^1.0.0-rc.12", "crypto-js": "^4.1.1", "form-data": "^4.0.0", - "node-fetch": "^3.3.2" + "node-fetch": "^2.7.0" } } diff --git a/run-source.ts b/run-source.ts new file mode 100644 index 0000000..0c86da0 --- /dev/null +++ b/run-source.ts @@ -0,0 +1,405 @@ +import nodeFetch from 'node-fetch'; +import { prompt } from 'enquirer'; +import Spinnies from 'spinnies'; +import { program } from 'commander'; +import dotenv from 'dotenv'; +import { makeProviders, targets, makeStandardFetcher, MovieMedia, ShowMedia, ProviderControls, MetaOutput } from '.'; + +dotenv.config(); + +type ProviderSourceAnswers = { + id: string; + type: string; +}; + +type EmbedSourceAnswers = { + url: string; +}; + +type CommonAnswers = { + fetcher: string; + source: string; +}; + +type ShowAnswers = { + season: string; + episode: string; +}; + +type CommandLineArguments = { + fetcher: string; + sourceId: string; + tmdbId: string; + type: string; + season: string; + episode: string; + url: string; +}; + +const TMDB_API_KEY = process.env.MOVIE_WEB_TMDB_API_KEY; + +const sources = getAllSources(); + +function getAllSources() { + // * The only way to get a list of all sources is to + // * create all these things. Maybe this should change + const nativeSources = makeProviders({ + fetcher: makeStandardFetcher(nodeFetch), + target: targets.NATIVE + }).listSources(); + + const browserSources = makeProviders({ + fetcher: makeStandardFetcher(nodeFetch), + target: targets.BROWSER + }).listSources(); + + const nativeEmbeds = makeProviders({ + fetcher: makeStandardFetcher(nodeFetch), + target: targets.NATIVE + }).listEmbeds(); + + const browserEmbeds = makeProviders({ + fetcher: makeStandardFetcher(nodeFetch), + target: targets.BROWSER + }).listEmbeds(); + + const combined = [ + ...nativeSources, ...browserSources, + ...nativeEmbeds, ...browserEmbeds + ]; + + // * Remove dupes + const map = new Map(combined.map(source => [source.id, source])); + + return [...map.values()] +} + +async function getMovieMediaDetails(id: string): Promise { + const response = await fetch(`https://api.themoviedb.org/3/movie/${id}?api_key=${TMDB_API_KEY}`, { + method: 'GET', + headers: { + accept: 'application/json' + } + }); + const movie = await response.json(); + + if (movie.success === false) { + throw new Error(movie.status_message); + } + + return { + type: 'movie', + title: movie.title, + releaseYear: Number(movie.release_date.split('-')[0]), + tmdbId: id + }; +} + +async function getShowMediaDetails(id: string, seasonNumber: string, episodeNumber: string): Promise { + // * TV shows require the TMDB ID for the series, season, and episode + // * and the name of the series. Needs multiple requests + let response = await fetch(`https://api.themoviedb.org/3/tv/${id}?api_key=${TMDB_API_KEY}`, { + method: 'GET', + headers: { + accept: 'application/json' + } + }); + const series = await response.json(); + + if (series.success === false) { + throw new Error(series.status_message); + } + + response = await fetch(`https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}?api_key=${TMDB_API_KEY}`, { + method: 'GET', + headers: { + accept: 'application/json' + } + }); + const season = await response.json(); + + if (season.success === false) { + throw new Error(season.status_message); + } + + response = await fetch(`https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}/episode/${episodeNumber}?api_key=${TMDB_API_KEY}`, { + method: 'GET', + headers: { + accept: 'application/json' + } + }); + const episode = await response.json(); + + if (episode.success === false) { + throw new Error(episode.status_message); + } + + return { + type: 'show', + title: series.name, + releaseYear: Number(series.first_air_date.split('-')[0]), // * Is this really what should go here? + tmdbId: id, + episode: { + number: episode.episode_number, + tmdbId: episode.id + }, + season: { + number: season.season_number, + tmdbId: season.id + } + }; +} + +function joinMediaTypes(mediaTypes: string[] | undefined) { + if (mediaTypes) { + const formatted = mediaTypes.map((type: string) => { + type = type[0].toUpperCase() + type.substring(1).toLowerCase(); + return `${type}s`; + }).join(' / '); + + return `(${formatted})`; + } else { + return ''; // * Embed sources pass through here too + } +} + +async function runQuestions() { + const options = { + fetcher: 'node-fetch', + sourceId: '', + tmdbId: '', + type: 'movie', + season: '0', + episode: '0', + url: '' + }; + + const answers = await prompt([ + { + type: 'select', + name: 'fetcher', + message: 'Select a fetcher', + choices: [ + { + + message: 'Native', + name: 'native' + }, + { + message: 'Node fetch', + name: 'node-fetch' + } + ] + }, + { + type: 'select', + name: 'source', + message: 'Select a source', + choices: sources.map(source => ({ + message: `[${source.type.toLocaleUpperCase()}] ${source.name} ${joinMediaTypes(source.mediaTypes)}`.trim(), + name: source.id + })) + } + ]); + + options.fetcher = answers.fetcher; + options.sourceId = answers.source; + + const source = sources.find(source => source.id === answers.source)!; + + if (source.type === 'embed') { + const sourceAnswers = await prompt([ + { + type: 'input', + name: 'url', + message: 'Embed URL' + } + ]); + + options.url = sourceAnswers.url; + } else { + const sourceAnswers = await prompt([ + { + type: 'input', + name: 'id', + message: 'TMDB ID' + }, + { + type: 'select', + name: 'type', + message: 'Media type', + choices: [ + { + message: 'Movie', + name: 'movie' + }, + { + message: 'TV Show', + name: 'show' + } + ] + } + ]); + + options.tmdbId = sourceAnswers.id; + options.type = sourceAnswers.type; + + if (sourceAnswers.type === 'show') { + const seriesAnswers = await prompt([ + { + type: 'input', + name: 'season', + message: 'Season' + }, + { + type: 'input', + name: 'episode', + message: 'Episode' + } + ]); + + options.season = seriesAnswers.season; + options.episode = seriesAnswers.episode; + } + } + + await processOptions(options); +} + +async function runCommandLine() { + program + .option('-f, --fetcher ', 'Fetcher to use. Currently only \'node-fetch\' is suspported', 'node-fetch') + .option('-sid, --source-id ', 'ID for the source to use. Either an embed or provider', '') + .option('-tid, --tmdb-id ', 'TMDB ID for the media to scrape. Only used if source is a provider', '') + .option('-t, --type ', 'Media type. Either \'movie\' or \'show\'. Only used if source is a provider', 'movie') + .option('-s, --season ', 'Season number. Only used if type is \'show\'', '0') + .option('-e, --episode ', 'Episode number. Only used if type is \'show\'', '0') + .option('-u, --url Promise' is not assignable to parameter of type 'typeof fetch'. + // * Property 'isRedirect' is missing in type '(input: RequestInfo | URL, init?: RequestInit | undefined) => Promise' but required in type 'typeof fetch'. ts(2345) + // * index.d.ts(221, 14): 'isRedirect' is declared here. + throw new Error('Fetcher must be node-fetch'); + } + + if (!options.sourceId.trim()) { + throw new Error('Source ID must be provided'); + } + + const source = sources.find(source => source.id === options.sourceId); + + if (!source) { + throw new Error('Invalid source ID. No source found'); + } + + if (source.type === 'embed' && !options.url.trim()) { + throw new Error('Must provide an embed URL for embed sources'); + } + + if (source.type === 'source') { + if (!options.tmdbId.trim()) { + throw new Error('Must provide a TMDB ID for provider sources'); + } + + if (isNaN(Number(options.tmdbId)) || Number(options.tmdbId) < 0) { + throw new Error('TMDB ID must be a number greater than 0'); + } + + if (!options.type.trim()) { + throw new Error('Must provide a type for provider sources'); + } + + if (options.type !== 'movie' && options.type !== 'show') { + throw new Error('Invalid media type. Must be either \'movie\' or \'show\''); + } + + if (options.type === 'show') { + if (!options.season.trim()) { + throw new Error('Must provide a season number for TV shows'); + } + + if (!options.episode.trim()) { + throw new Error('Must provide an episode number for TV shows'); + } + + if (isNaN(Number(options.season)) || Number(options.season) <= 0) { + throw new Error('Season number must be a number greater than 0'); + } + + if (isNaN(Number(options.episode)) || Number(options.episode) <= 0) { + throw new Error('Episode number must be a number greater than 0'); + } + } + } + + const fetcher = makeStandardFetcher(nodeFetch); + const providers = makeProviders({ + fetcher: fetcher, + target: targets.NATIVE + }); + + await runScraper(providers, source, options); +} + +async function runScraper(providers: ProviderControls, source: MetaOutput, options: CommandLineArguments) { + const spinnies = new Spinnies(); + + if (source.type === 'embed') { + spinnies.add('scrape', { text: `Running ${source.name} scraper on ${options.url}` }); + try { + const result = await providers.runEmbedScraper({ + url: options.url, + id: source.id + }); + spinnies.succeed('scrape', { text: 'Done!' }); + console.log(result); + } catch (error) { + let message = 'Unknown error'; + if (error instanceof Error) { + message = error.message; + } + + spinnies.fail('scrape', { text: `ERROR: ${message}` }); + } + } else { + let media; + + if (options.type === 'movie') { + media = await getMovieMediaDetails(options.tmdbId); + } else { + media = await getShowMediaDetails(options.tmdbId, options.season, options.episode); + } + + spinnies.add('scrape', { text: `Running ${source.name} scraper on ${media.title}` }); + try { + const result = await providers.runSourceScraper({ + media: media, + id: source.id + }); + spinnies.succeed('scrape', { text: 'Done!' }); + console.log(result); + } catch (error) { + let message = 'Unknown error'; + if (error instanceof Error) { + message = error.message; + } + + spinnies.fail('scrape', { text: `ERROR: ${message}` }); + } + } +} + +if (process.argv.length === 2) { + runQuestions(); +} else { + runCommandLine(); +} diff --git a/tsconfig.json b/tsconfig.json index 4ea9f0b..a476bbc 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "target": "es2021", - "lib": ["es2021"], + "lib": ["es2021", "dom"], "module": "CommonJS", "esModuleInterop": true, "declaration": true, From a9a4dc0b21494e31973c17d53c8b065f56ff5cc9 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Tue, 26 Sep 2023 02:25:04 -0400 Subject: [PATCH 02/17] Added native fetch to source tester --- run-source.ts | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/run-source.ts b/run-source.ts index 0c86da0..e2a25be 100644 --- a/run-source.ts +++ b/run-source.ts @@ -268,7 +268,7 @@ async function runQuestions() { async function runCommandLine() { program - .option('-f, --fetcher ', 'Fetcher to use. Currently only \'node-fetch\' is suspported', 'node-fetch') + .option('-f, --fetcher ', 'Fetcher to use. Either \'native\' \'node-fetch\'', 'node-fetch') .option('-sid, --source-id ', 'ID for the source to use. Either an embed or provider', '') .option('-tid, --tmdb-id ', 'TMDB ID for the media to scrape. Only used if source is a provider', '') .option('-t, --type ', 'Media type. Either \'movie\' or \'show\'. Only used if source is a provider', 'movie') @@ -282,13 +282,8 @@ async function runCommandLine() { } async function processOptions(options: CommandLineArguments) { - if (options.fetcher !== 'node-fetch') { - // * Using native fetch in makeStandardFetcher throws the following TS error: - // * - // * Argument of type '(input: RequestInfo | URL, init?: RequestInit | undefined) => Promise' is not assignable to parameter of type 'typeof fetch'. - // * Property 'isRedirect' is missing in type '(input: RequestInfo | URL, init?: RequestInit | undefined) => Promise' but required in type 'typeof fetch'. ts(2345) - // * index.d.ts(221, 14): 'isRedirect' is declared here. - throw new Error('Fetcher must be node-fetch'); + if (options.fetcher !== 'node-fetch' && options.fetcher !== 'native') { + throw new Error('Fetcher must be either \'native\' or \'node-fetch\''); } if (!options.sourceId.trim()) { @@ -341,7 +336,14 @@ async function processOptions(options: CommandLineArguments) { } } - const fetcher = makeStandardFetcher(nodeFetch); + let fetcher; + + if (options.fetcher === 'native') { + fetcher = makeStandardFetcher(fetch as any); + } else { + fetcher = makeStandardFetcher(nodeFetch); + } + const providers = makeProviders({ fetcher: fetcher, target: targets.NATIVE From a1e12d91a1ef9d21ba5d0cedaf4c95093ba2a9ed Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Tue, 26 Sep 2023 02:26:03 -0400 Subject: [PATCH 03/17] Typo in --fetcher details --- run-source.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run-source.ts b/run-source.ts index e2a25be..f7872af 100644 --- a/run-source.ts +++ b/run-source.ts @@ -268,7 +268,7 @@ async function runQuestions() { async function runCommandLine() { program - .option('-f, --fetcher ', 'Fetcher to use. Either \'native\' \'node-fetch\'', 'node-fetch') + .option('-f, --fetcher ', 'Fetcher to use. Either \'native\' or \'node-fetch\'', 'node-fetch') .option('-sid, --source-id ', 'ID for the source to use. Either an embed or provider', '') .option('-tid, --tmdb-id ', 'TMDB ID for the media to scrape. Only used if source is a provider', '') .option('-t, --type ', 'Media type. Either \'movie\' or \'show\'. Only used if source is a provider', 'movie') From 72bd5677a2cc8ee1d58f4716f4c292183f09a4f2 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 26 Sep 2023 18:34:52 +0200 Subject: [PATCH 04/17] add ts-node into dev deps --- package-lock.json | 141 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 2 + 2 files changed, 143 insertions(+) diff --git a/package-lock.json b/package-lock.json index 26d6d86..6606ff1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,6 +32,7 @@ "eslint-plugin-prettier": "^4.2.1", "prettier": "^2.6.2", "spinnies": "^0.5.1", + "ts-node": "^10.9.1", "tsc-alias": "^1.6.7", "typescript": "^4.6.3", "vite": "^4.0.0", @@ -71,6 +72,28 @@ "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", "dev": true }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@cspotcode/source-map-support/node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, "node_modules/@esbuild/android-arm": { "version": "0.17.19", "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.17.19.tgz", @@ -816,6 +839,30 @@ "node": ">= 6" } }, + "node_modules/@tsconfig/node10": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", + "integrity": "sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==", + "dev": true + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true + }, + "node_modules/@tsconfig/node16": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", + "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", + "dev": true + }, "node_modules/@types/argparse": { "version": "1.0.38", "resolved": "https://registry.npmjs.org/@types/argparse/-/argparse-1.0.38.tgz", @@ -1623,6 +1670,12 @@ "node": ">= 8" } }, + "node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -2095,6 +2148,12 @@ "integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==", "dev": true }, + "node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -2427,6 +2486,15 @@ "node": ">=0.4.0" } }, + "node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true, + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -4484,6 +4552,12 @@ "node": ">=10" } }, + "node_modules/make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true + }, "node_modules/md5-hex": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/md5-hex/-/md5-hex-3.0.1.tgz", @@ -5842,6 +5916,58 @@ "node": ">=8" } }, + "node_modules/ts-node": { + "version": "10.9.1", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", + "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", + "dev": true, + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, + "node_modules/ts-node/node_modules/acorn-walk": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz", + "integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==", + "dev": true, + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/tsc-alias": { "version": "1.8.6", "resolved": "https://registry.npmjs.org/tsc-alias/-/tsc-alias-1.8.6.tgz", @@ -5996,6 +6122,12 @@ "requires-port": "^1.0.0" } }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true + }, "node_modules/v8-to-istanbul": { "version": "9.1.0", "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.1.0.tgz", @@ -6517,6 +6649,15 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", "dev": true }, + "node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index 8ce3f6d..35676f6 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "scripts": { "build": "vite build", "test": "vitest run", + "test:dev": "ts-node ./run-source.ts", "test:watch": "vitest", "test:coverage": "vitest run --coverage", "lint": "eslint --ext .ts,.js src/", @@ -62,6 +63,7 @@ "eslint-plugin-prettier": "^4.2.1", "prettier": "^2.6.2", "spinnies": "^0.5.1", + "ts-node": "^10.9.1", "tsc-alias": "^1.6.7", "typescript": "^4.6.3", "vite": "^4.0.0", From 18cd9141a24a063335982752fe8703e8f58b86d4 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 26 Sep 2023 18:45:21 +0200 Subject: [PATCH 05/17] update dom lib --- tsconfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tsconfig.json b/tsconfig.json index a476bbc..3c5f515 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "target": "es2021", - "lib": ["es2021", "dom"], + "lib": ["es2021", "DOM"], "module": "CommonJS", "esModuleInterop": true, "declaration": true, From 6ca9de8d747518345583b0e4c08f6a923a0f26f5 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Tue, 26 Sep 2023 13:18:41 -0400 Subject: [PATCH 06/17] missing > in --url --- run-source.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run-source.ts b/run-source.ts index f7872af..3f53106 100644 --- a/run-source.ts +++ b/run-source.ts @@ -274,7 +274,7 @@ async function runCommandLine() { .option('-t, --type ', 'Media type. Either \'movie\' or \'show\'. Only used if source is a provider', 'movie') .option('-s, --season ', 'Season number. Only used if type is \'show\'', '0') .option('-e, --episode ', 'Episode number. Only used if type is \'show\'', '0') - .option('-u, --url ', 'URL to a video embed. Only used if source is an embed', '') program.parse(); From 6f29b636ead81d7252ac2eacbe61b3c120a91cdd Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Tue, 26 Sep 2023 13:24:16 -0400 Subject: [PATCH 07/17] Removed duplicate code in getAllSources --- run-source.ts | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/run-source.ts b/run-source.ts index 3f53106..0be2667 100644 --- a/run-source.ts +++ b/run-source.ts @@ -43,30 +43,15 @@ const sources = getAllSources(); function getAllSources() { // * The only way to get a list of all sources is to // * create all these things. Maybe this should change - const nativeSources = makeProviders({ + const providers = makeProviders({ fetcher: makeStandardFetcher(nodeFetch), target: targets.NATIVE - }).listSources(); + }) - const browserSources = makeProviders({ - fetcher: makeStandardFetcher(nodeFetch), - target: targets.BROWSER - }).listSources(); + const sources = providers.listSources(); + const embeds = providers.listEmbeds(); - const nativeEmbeds = makeProviders({ - fetcher: makeStandardFetcher(nodeFetch), - target: targets.NATIVE - }).listEmbeds(); - - const browserEmbeds = makeProviders({ - fetcher: makeStandardFetcher(nodeFetch), - target: targets.BROWSER - }).listEmbeds(); - - const combined = [ - ...nativeSources, ...browserSources, - ...nativeEmbeds, ...browserEmbeds - ]; + const combined = [...sources, ...embeds]; // * Remove dupes const map = new Map(combined.map(source => [source.id, source])); From 7804b3bf917468ff29faeb8416a273f2ec184b39 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Tue, 26 Sep 2023 13:25:54 -0400 Subject: [PATCH 08/17] throw error if missing MOVIE_WEB_TMDB_API_KEY --- run-source.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/run-source.ts b/run-source.ts index 0be2667..4a24d81 100644 --- a/run-source.ts +++ b/run-source.ts @@ -38,6 +38,10 @@ type CommandLineArguments = { const TMDB_API_KEY = process.env.MOVIE_WEB_TMDB_API_KEY; +if (!TMDB_API_KEY?.trim()) { + throw new Error('Missing MOVIE_WEB_TMDB_API_KEY environment variable'); +} + const sources = getAllSources(); function getAllSources() { From 5af235ff748bb38385908da11e03bc3a8dc94f71 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Tue, 26 Sep 2023 13:29:54 -0400 Subject: [PATCH 09/17] throw if media has not been released yet --- run-source.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/run-source.ts b/run-source.ts index 4a24d81..7ee6c04 100644 --- a/run-source.ts +++ b/run-source.ts @@ -76,6 +76,10 @@ async function getMovieMediaDetails(id: string): Promise { throw new Error(movie.status_message); } + if (!movie.release_date) { + throw new Error(`${movie.title} has no release_date. Assuming unreleased`); + } + return { type: 'movie', title: movie.title, @@ -99,6 +103,10 @@ async function getShowMediaDetails(id: string, seasonNumber: string, episodeNumb throw new Error(series.status_message); } + if (!series.first_air_date) { + throw new Error(`${series.name} has no first_air_date. Assuming unaired`); + } + response = await fetch(`https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}?api_key=${TMDB_API_KEY}`, { method: 'GET', headers: { @@ -126,7 +134,7 @@ async function getShowMediaDetails(id: string, seasonNumber: string, episodeNumb return { type: 'show', title: series.name, - releaseYear: Number(series.first_air_date.split('-')[0]), // * Is this really what should go here? + releaseYear: Number(series.first_air_date.split('-')[0]), tmdbId: id, episode: { number: episode.episode_number, From e008ebd01940dc041ce8555ed3bd00251b4161a0 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Tue, 26 Sep 2023 13:47:30 -0400 Subject: [PATCH 10/17] support all API key types --- run-source.ts | 49 ++++++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/run-source.ts b/run-source.ts index 7ee6c04..b159180 100644 --- a/run-source.ts +++ b/run-source.ts @@ -63,13 +63,31 @@ function getAllSources() { return [...map.values()] } -async function getMovieMediaDetails(id: string): Promise { - const response = await fetch(`https://api.themoviedb.org/3/movie/${id}?api_key=${TMDB_API_KEY}`, { +async function makeTMDBRequest(url: string): Promise { + const headers: { + accept: 'application/json'; + authorization?: string; + } = { + accept: 'application/json' + }; + + // * JWT keys always start with ey and are ONLY valid as a header. + // * All other keys are ONLY valid as a query param. + // * Thanks TMDB. + if (TMDB_API_KEY!.startsWith('ey')) { + headers.authorization = `Bearer ${TMDB_API_KEY}`; + } else { + url += `?api_key=${TMDB_API_KEY}`; + } + + return fetch(url, { method: 'GET', - headers: { - accept: 'application/json' - } + headers: headers }); +} + +async function getMovieMediaDetails(id: string): Promise { + const response = await makeTMDBRequest(`https://api.themoviedb.org/3/movie/${id}`); const movie = await response.json(); if (movie.success === false) { @@ -91,12 +109,7 @@ async function getMovieMediaDetails(id: string): Promise { async function getShowMediaDetails(id: string, seasonNumber: string, episodeNumber: string): Promise { // * TV shows require the TMDB ID for the series, season, and episode // * and the name of the series. Needs multiple requests - let response = await fetch(`https://api.themoviedb.org/3/tv/${id}?api_key=${TMDB_API_KEY}`, { - method: 'GET', - headers: { - accept: 'application/json' - } - }); + let response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}`); const series = await response.json(); if (series.success === false) { @@ -107,24 +120,14 @@ async function getShowMediaDetails(id: string, seasonNumber: string, episodeNumb throw new Error(`${series.name} has no first_air_date. Assuming unaired`); } - response = await fetch(`https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}?api_key=${TMDB_API_KEY}`, { - method: 'GET', - headers: { - accept: 'application/json' - } - }); + response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}`); const season = await response.json(); if (season.success === false) { throw new Error(season.status_message); } - response = await fetch(`https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}/episode/${episodeNumber}?api_key=${TMDB_API_KEY}`, { - method: 'GET', - headers: { - accept: 'application/json' - } - }); + response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}/episode/${episodeNumber}`); const episode = await response.json(); if (episode.success === false) { From b70a27a0c8be16652ebc871496ce800b35243c1b Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Tue, 26 Sep 2023 14:10:02 -0400 Subject: [PATCH 11/17] move script into src --- package.json | 2 +- run-source.ts => src/dev-cli.ts | 119 +++++++++++++++++--------------- 2 files changed, 64 insertions(+), 57 deletions(-) rename run-source.ts => src/dev-cli.ts (79%) diff --git a/package.json b/package.json index 35676f6..17ac9e3 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "scripts": { "build": "vite build", "test": "vitest run", - "test:dev": "ts-node ./run-source.ts", + "test:dev": "npx ts-node ./src/dev-cli.ts", "test:watch": "vitest", "test:coverage": "vitest run --coverage", "lint": "eslint --ext .ts,.js src/", diff --git a/run-source.ts b/src/dev-cli.ts similarity index 79% rename from run-source.ts rename to src/dev-cli.ts index b159180..1ec3528 100644 --- a/run-source.ts +++ b/src/dev-cli.ts @@ -1,9 +1,14 @@ -import nodeFetch from 'node-fetch'; -import { prompt } from 'enquirer'; -import Spinnies from 'spinnies'; +// eslint-disable-next-line import/no-extraneous-dependencies import { program } from 'commander'; +// eslint-disable-next-line import/no-extraneous-dependencies import dotenv from 'dotenv'; -import { makeProviders, targets, makeStandardFetcher, MovieMedia, ShowMedia, ProviderControls, MetaOutput } from '.'; +// eslint-disable-next-line import/no-extraneous-dependencies +import { prompt } from 'enquirer'; +import nodeFetch from 'node-fetch'; +// eslint-disable-next-line import/no-extraneous-dependencies +import Spinnies from 'spinnies'; + +import { MetaOutput, MovieMedia, ProviderControls, ShowMedia, makeProviders, makeStandardFetcher, targets } from '..'; dotenv.config(); @@ -49,8 +54,8 @@ function getAllSources() { // * create all these things. Maybe this should change const providers = makeProviders({ fetcher: makeStandardFetcher(nodeFetch), - target: targets.NATIVE - }) + target: targets.NATIVE, + }); const sources = providers.listSources(); const embeds = providers.listEmbeds(); @@ -58,9 +63,9 @@ function getAllSources() { const combined = [...sources, ...embeds]; // * Remove dupes - const map = new Map(combined.map(source => [source.id, source])); + const map = new Map(combined.map((source) => [source.id, source])); - return [...map.values()] + return [...map.values()]; } async function makeTMDBRequest(url: string): Promise { @@ -68,7 +73,7 @@ async function makeTMDBRequest(url: string): Promise { accept: 'application/json'; authorization?: string; } = { - accept: 'application/json' + accept: 'application/json', }; // * JWT keys always start with ey and are ONLY valid as a header. @@ -82,7 +87,7 @@ async function makeTMDBRequest(url: string): Promise { return fetch(url, { method: 'GET', - headers: headers + headers, }); } @@ -102,7 +107,7 @@ async function getMovieMediaDetails(id: string): Promise { type: 'movie', title: movie.title, releaseYear: Number(movie.release_date.split('-')[0]), - tmdbId: id + tmdbId: id, }; } @@ -127,7 +132,9 @@ async function getShowMediaDetails(id: string, seasonNumber: string, episodeNumb throw new Error(season.status_message); } - response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}/episode/${episodeNumber}`); + response = await makeTMDBRequest( + `https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}/episode/${episodeNumber}`, + ); const episode = await response.json(); if (episode.success === false) { @@ -141,26 +148,27 @@ async function getShowMediaDetails(id: string, seasonNumber: string, episodeNumb tmdbId: id, episode: { number: episode.episode_number, - tmdbId: episode.id + tmdbId: episode.id, }, season: { number: season.season_number, - tmdbId: season.id - } + tmdbId: season.id, + }, }; } function joinMediaTypes(mediaTypes: string[] | undefined) { if (mediaTypes) { - const formatted = mediaTypes.map((type: string) => { - type = type[0].toUpperCase() + type.substring(1).toLowerCase(); - return `${type}s`; - }).join(' / '); + const formatted = mediaTypes + .map((type: string) => { + type = type[0].toUpperCase() + type.substring(1).toLowerCase(); + return `${type}s`; + }) + .join(' / '); return `(${formatted})`; - } else { - return ''; // * Embed sources pass through here too } + return ''; // * Embed sources pass through here too } async function runQuestions() { @@ -171,7 +179,7 @@ async function runQuestions() { type: 'movie', season: '0', episode: '0', - url: '' + url: '', }; const answers = await prompt([ @@ -181,39 +189,38 @@ async function runQuestions() { message: 'Select a fetcher', choices: [ { - message: 'Native', - name: 'native' + name: 'native', }, { message: 'Node fetch', - name: 'node-fetch' - } - ] + name: 'node-fetch', + }, + ], }, { type: 'select', name: 'source', message: 'Select a source', - choices: sources.map(source => ({ + choices: sources.map((source) => ({ message: `[${source.type.toLocaleUpperCase()}] ${source.name} ${joinMediaTypes(source.mediaTypes)}`.trim(), - name: source.id - })) - } + name: source.id, + })), + }, ]); options.fetcher = answers.fetcher; options.sourceId = answers.source; - const source = sources.find(source => source.id === answers.source)!; + const source = sources.find((source) => source.id === answers.source)!; if (source.type === 'embed') { const sourceAnswers = await prompt([ { type: 'input', name: 'url', - message: 'Embed URL' - } + message: 'Embed URL', + }, ]); options.url = sourceAnswers.url; @@ -222,7 +229,7 @@ async function runQuestions() { { type: 'input', name: 'id', - message: 'TMDB ID' + message: 'TMDB ID', }, { type: 'select', @@ -231,14 +238,14 @@ async function runQuestions() { choices: [ { message: 'Movie', - name: 'movie' + name: 'movie', }, { message: 'TV Show', - name: 'show' - } - ] - } + name: 'show', + }, + ], + }, ]); options.tmdbId = sourceAnswers.id; @@ -249,13 +256,13 @@ async function runQuestions() { { type: 'input', name: 'season', - message: 'Season' + message: 'Season', }, { type: 'input', name: 'episode', - message: 'Episode' - } + message: 'Episode', + }, ]); options.season = seriesAnswers.season; @@ -268,13 +275,13 @@ async function runQuestions() { async function runCommandLine() { program - .option('-f, --fetcher ', 'Fetcher to use. Either \'native\' or \'node-fetch\'', 'node-fetch') + .option('-f, --fetcher ', "Fetcher to use. Either 'native' or 'node-fetch'", 'node-fetch') .option('-sid, --source-id ', 'ID for the source to use. Either an embed or provider', '') .option('-tid, --tmdb-id ', 'TMDB ID for the media to scrape. Only used if source is a provider', '') - .option('-t, --type ', 'Media type. Either \'movie\' or \'show\'. Only used if source is a provider', 'movie') - .option('-s, --season ', 'Season number. Only used if type is \'show\'', '0') - .option('-e, --episode ', 'Episode number. Only used if type is \'show\'', '0') - .option('-u, --url ', 'URL to a video embed. Only used if source is an embed', '') + .option('-t, --type ', "Media type. Either 'movie' or 'show'. Only used if source is a provider", 'movie') + .option('-s, --season ', "Season number. Only used if type is 'show'", '0') + .option('-e, --episode ', "Episode number. Only used if type is 'show'", '0') + .option('-u, --url ', 'URL to a video embed. Only used if source is an embed', ''); program.parse(); @@ -283,14 +290,14 @@ async function runCommandLine() { async function processOptions(options: CommandLineArguments) { if (options.fetcher !== 'node-fetch' && options.fetcher !== 'native') { - throw new Error('Fetcher must be either \'native\' or \'node-fetch\''); + throw new Error("Fetcher must be either 'native' or 'node-fetch'"); } if (!options.sourceId.trim()) { throw new Error('Source ID must be provided'); } - const source = sources.find(source => source.id === options.sourceId); + const source = sources.find((source) => source.id === options.sourceId); if (!source) { throw new Error('Invalid source ID. No source found'); @@ -314,7 +321,7 @@ async function processOptions(options: CommandLineArguments) { } if (options.type !== 'movie' && options.type !== 'show') { - throw new Error('Invalid media type. Must be either \'movie\' or \'show\''); + throw new Error("Invalid media type. Must be either 'movie' or 'show'"); } if (options.type === 'show') { @@ -345,8 +352,8 @@ async function processOptions(options: CommandLineArguments) { } const providers = makeProviders({ - fetcher: fetcher, - target: targets.NATIVE + fetcher, + target: targets.NATIVE, }); await runScraper(providers, source, options); @@ -360,7 +367,7 @@ async function runScraper(providers: ProviderControls, source: MetaOutput, optio try { const result = await providers.runEmbedScraper({ url: options.url, - id: source.id + id: source.id, }); spinnies.succeed('scrape', { text: 'Done!' }); console.log(result); @@ -384,8 +391,8 @@ async function runScraper(providers: ProviderControls, source: MetaOutput, optio spinnies.add('scrape', { text: `Running ${source.name} scraper on ${media.title}` }); try { const result = await providers.runSourceScraper({ - media: media, - id: source.id + media, + id: source.id, }); spinnies.succeed('scrape', { text: 'Done!' }); console.log(result); From 404c48f3b362a0a4ae9ec9be878c3c63edd65ae8 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Tue, 26 Sep 2023 14:21:18 -0400 Subject: [PATCH 12/17] updated README to mention the test script and arguments --- README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/README.md b/README.md index 1aa330b..f6178d0 100644 --- a/README.md +++ b/README.md @@ -9,5 +9,30 @@ features: > **This package is still WIP** +## Development +To make testing scrapers easier during development a CLI tool is available to run specific sources. To run the CLI testing tool, use `npm run test:dev`. The script supports 2 execution modes + +- CLI Mode, for passing in arguments directly to the script +- Question Mode, where the script asks you questions about which source you wish to test + +The following CLI Mode arguments are available + +| Argument | Alias | Description | Default | +|---------------|--------|-------------------------------------------------------------------------|--------------| +| `--fetcher` | `-f` | Fetcher type. Either `node-fetch` or `native` | `node-fetch` | +| `--source-id` | `-sid` | Source ID for the source to be tested | | +| `--tmdb-id` | `-tid` | TMDB ID for the media to scrape. Only used if source is a provider | | +| `--type` | `-t` | Media type. Either `movie` or `show`. Only used if source is a provider | `movie` | +| `--season` | `-s` | Season number. Only used if type is `show` | `0` | +| `--episode` | `-e` | Episode number. Only used if type is `show` | `0` | +| `--url` | `-u` | URL to a video embed. Only used if source is an embed | | +| `--help` | `-h` | Shows help for the command arguments | | + +Example testing the FlixHQ source on the movie "Spirited Away" + +```bash +npm run test:dev -- -sid flixhq -tid 129 -t movie +``` + Todos: - make default fetcher maker thing work with both undici and node-fetch From 688d0c008a890506f1ad96cc56e1f047784c902e Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Tue, 26 Sep 2023 14:26:10 -0400 Subject: [PATCH 13/17] eslint fixes --- src/dev-cli.ts | 261 +++++++++++++++++++++++++------------------------ 1 file changed, 131 insertions(+), 130 deletions(-) diff --git a/src/dev-cli.ts b/src/dev-cli.ts index 1ec3528..ca3e6b6 100644 --- a/src/dev-cli.ts +++ b/src/dev-cli.ts @@ -47,8 +47,6 @@ if (!TMDB_API_KEY?.trim()) { throw new Error('Missing MOVIE_WEB_TMDB_API_KEY environment variable'); } -const sources = getAllSources(); - function getAllSources() { // * The only way to get a list of all sources is to // * create all these things. Maybe this should change @@ -57,10 +55,7 @@ function getAllSources() { target: targets.NATIVE, }); - const sources = providers.listSources(); - const embeds = providers.listEmbeds(); - - const combined = [...sources, ...embeds]; + const combined = [...providers.listSources(), ...providers.listEmbeds()]; // * Remove dupes const map = new Map(combined.map((source) => [source.id, source])); @@ -68,6 +63,9 @@ function getAllSources() { return [...map.values()]; } +// * Defined here cuz ESLint didn't like the order these were defined in +const sources = getAllSources(); + async function makeTMDBRequest(url: string): Promise { const headers: { accept: 'application/json'; @@ -76,16 +74,20 @@ async function makeTMDBRequest(url: string): Promise { accept: 'application/json', }; + // * Used to get around ESLint + // * Assignment to function parameter 'url'. eslint (no-param-reassign) + let requestURL = url; + // * JWT keys always start with ey and are ONLY valid as a header. // * All other keys are ONLY valid as a query param. // * Thanks TMDB. if (TMDB_API_KEY!.startsWith('ey')) { headers.authorization = `Bearer ${TMDB_API_KEY}`; } else { - url += `?api_key=${TMDB_API_KEY}`; + requestURL += `?api_key=${TMDB_API_KEY}`; } - return fetch(url, { + return fetch(requestURL, { method: 'GET', headers, }); @@ -161,8 +163,7 @@ function joinMediaTypes(mediaTypes: string[] | undefined) { if (mediaTypes) { const formatted = mediaTypes .map((type: string) => { - type = type[0].toUpperCase() + type.substring(1).toLowerCase(); - return `${type}s`; + return `${type[0].toUpperCase() + type.substring(1).toLowerCase()}s`; }) .join(' / '); @@ -171,6 +172,125 @@ function joinMediaTypes(mediaTypes: string[] | undefined) { return ''; // * Embed sources pass through here too } +async function runScraper(providers: ProviderControls, source: MetaOutput, options: CommandLineArguments) { + const spinnies = new Spinnies(); + + if (source.type === 'embed') { + spinnies.add('scrape', { text: `Running ${source.name} scraper on ${options.url}` }); + try { + const result = await providers.runEmbedScraper({ + url: options.url, + id: source.id, + }); + spinnies.succeed('scrape', { text: 'Done!' }); + console.log(result); + } catch (error) { + let message = 'Unknown error'; + if (error instanceof Error) { + message = error.message; + } + + spinnies.fail('scrape', { text: `ERROR: ${message}` }); + } + } else { + let media; + + if (options.type === 'movie') { + media = await getMovieMediaDetails(options.tmdbId); + } else { + media = await getShowMediaDetails(options.tmdbId, options.season, options.episode); + } + + spinnies.add('scrape', { text: `Running ${source.name} scraper on ${media.title}` }); + try { + const result = await providers.runSourceScraper({ + media, + id: source.id, + }); + spinnies.succeed('scrape', { text: 'Done!' }); + console.log(result); + } catch (error) { + let message = 'Unknown error'; + if (error instanceof Error) { + message = error.message; + } + + spinnies.fail('scrape', { text: `ERROR: ${message}` }); + } + } +} + +async function processOptions(options: CommandLineArguments) { + if (options.fetcher !== 'node-fetch' && options.fetcher !== 'native') { + throw new Error("Fetcher must be either 'native' or 'node-fetch'"); + } + + if (!options.sourceId.trim()) { + throw new Error('Source ID must be provided'); + } + + const source = sources.find(({ id }) => id === options.sourceId); + + if (!source) { + throw new Error('Invalid source ID. No source found'); + } + + if (source.type === 'embed' && !options.url.trim()) { + throw new Error('Must provide an embed URL for embed sources'); + } + + if (source.type === 'source') { + if (!options.tmdbId.trim()) { + throw new Error('Must provide a TMDB ID for provider sources'); + } + + if (Number.isNaN(Number(options.tmdbId)) || Number(options.tmdbId) < 0) { + throw new Error('TMDB ID must be a number greater than 0'); + } + + if (!options.type.trim()) { + throw new Error('Must provide a type for provider sources'); + } + + if (options.type !== 'movie' && options.type !== 'show') { + throw new Error("Invalid media type. Must be either 'movie' or 'show'"); + } + + if (options.type === 'show') { + if (!options.season.trim()) { + throw new Error('Must provide a season number for TV shows'); + } + + if (!options.episode.trim()) { + throw new Error('Must provide an episode number for TV shows'); + } + + if (Number.isNaN(Number(options.season)) || Number(options.season) <= 0) { + throw new Error('Season number must be a number greater than 0'); + } + + if (Number.isNaN(Number(options.episode)) || Number(options.episode) <= 0) { + throw new Error('Episode number must be a number greater than 0'); + } + } + } + + let fetcher; + + if (options.fetcher === 'native') { + fetcher = makeStandardFetcher(fetch as any); + } else { + fetcher = makeStandardFetcher(nodeFetch); + } + + const providers = makeProviders({ + fetcher, + target: targets.NATIVE, + }); + + await runScraper(providers, source, options); +} + async function runQuestions() { const options = { fetcher: 'node-fetch', @@ -212,7 +332,7 @@ async function runQuestions() { options.fetcher = answers.fetcher; options.sourceId = answers.source; - const source = sources.find((source) => source.id === answers.source)!; + const source = sources.find(({ id }) => id === answers.source)!; if (source.type === 'embed') { const sourceAnswers = await prompt([ @@ -288,125 +408,6 @@ async function runCommandLine() { await processOptions(program.opts()); } -async function processOptions(options: CommandLineArguments) { - if (options.fetcher !== 'node-fetch' && options.fetcher !== 'native') { - throw new Error("Fetcher must be either 'native' or 'node-fetch'"); - } - - if (!options.sourceId.trim()) { - throw new Error('Source ID must be provided'); - } - - const source = sources.find((source) => source.id === options.sourceId); - - if (!source) { - throw new Error('Invalid source ID. No source found'); - } - - if (source.type === 'embed' && !options.url.trim()) { - throw new Error('Must provide an embed URL for embed sources'); - } - - if (source.type === 'source') { - if (!options.tmdbId.trim()) { - throw new Error('Must provide a TMDB ID for provider sources'); - } - - if (isNaN(Number(options.tmdbId)) || Number(options.tmdbId) < 0) { - throw new Error('TMDB ID must be a number greater than 0'); - } - - if (!options.type.trim()) { - throw new Error('Must provide a type for provider sources'); - } - - if (options.type !== 'movie' && options.type !== 'show') { - throw new Error("Invalid media type. Must be either 'movie' or 'show'"); - } - - if (options.type === 'show') { - if (!options.season.trim()) { - throw new Error('Must provide a season number for TV shows'); - } - - if (!options.episode.trim()) { - throw new Error('Must provide an episode number for TV shows'); - } - - if (isNaN(Number(options.season)) || Number(options.season) <= 0) { - throw new Error('Season number must be a number greater than 0'); - } - - if (isNaN(Number(options.episode)) || Number(options.episode) <= 0) { - throw new Error('Episode number must be a number greater than 0'); - } - } - } - - let fetcher; - - if (options.fetcher === 'native') { - fetcher = makeStandardFetcher(fetch as any); - } else { - fetcher = makeStandardFetcher(nodeFetch); - } - - const providers = makeProviders({ - fetcher, - target: targets.NATIVE, - }); - - await runScraper(providers, source, options); -} - -async function runScraper(providers: ProviderControls, source: MetaOutput, options: CommandLineArguments) { - const spinnies = new Spinnies(); - - if (source.type === 'embed') { - spinnies.add('scrape', { text: `Running ${source.name} scraper on ${options.url}` }); - try { - const result = await providers.runEmbedScraper({ - url: options.url, - id: source.id, - }); - spinnies.succeed('scrape', { text: 'Done!' }); - console.log(result); - } catch (error) { - let message = 'Unknown error'; - if (error instanceof Error) { - message = error.message; - } - - spinnies.fail('scrape', { text: `ERROR: ${message}` }); - } - } else { - let media; - - if (options.type === 'movie') { - media = await getMovieMediaDetails(options.tmdbId); - } else { - media = await getShowMediaDetails(options.tmdbId, options.season, options.episode); - } - - spinnies.add('scrape', { text: `Running ${source.name} scraper on ${media.title}` }); - try { - const result = await providers.runSourceScraper({ - media, - id: source.id, - }); - spinnies.succeed('scrape', { text: 'Done!' }); - console.log(result); - } catch (error) { - let message = 'Unknown error'; - if (error instanceof Error) { - message = error.message; - } - - spinnies.fail('scrape', { text: `ERROR: ${message}` }); - } - } -} - if (process.argv.length === 2) { runQuestions(); } else { From 88b2e631f07c4967a23f8fd982eb60b95165d4a5 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Tue, 26 Sep 2023 15:21:28 -0400 Subject: [PATCH 14/17] removed npx from test npm script --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 17ac9e3..d46ab74 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "scripts": { "build": "vite build", "test": "vitest run", - "test:dev": "npx ts-node ./src/dev-cli.ts", + "test:dev": "ts-node ./src/dev-cli.ts", "test:watch": "vitest", "test:coverage": "vitest run --coverage", "lint": "eslint --ext .ts,.js src/", From 147d2f3ea95eb19dd9e2d460f5b323eafc53a2db Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Tue, 26 Sep 2023 15:22:13 -0400 Subject: [PATCH 15/17] coalesce API key to remove linter warning --- src/dev-cli.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dev-cli.ts b/src/dev-cli.ts index ca3e6b6..78cd4dd 100644 --- a/src/dev-cli.ts +++ b/src/dev-cli.ts @@ -41,7 +41,7 @@ type CommandLineArguments = { url: string; }; -const TMDB_API_KEY = process.env.MOVIE_WEB_TMDB_API_KEY; +const TMDB_API_KEY = process.env.MOVIE_WEB_TMDB_API_KEY ?? ''; if (!TMDB_API_KEY?.trim()) { throw new Error('Missing MOVIE_WEB_TMDB_API_KEY environment variable'); @@ -81,7 +81,7 @@ async function makeTMDBRequest(url: string): Promise { // * JWT keys always start with ey and are ONLY valid as a header. // * All other keys are ONLY valid as a query param. // * Thanks TMDB. - if (TMDB_API_KEY!.startsWith('ey')) { + if (TMDB_API_KEY.startsWith('ey')) { headers.authorization = `Bearer ${TMDB_API_KEY}`; } else { requestURL += `?api_key=${TMDB_API_KEY}`; From ad0c25f68b15fd6a51884420b9d48fcc101faede Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Tue, 26 Sep 2023 15:34:29 -0400 Subject: [PATCH 16/17] changed script import path to skip compilation step --- package-lock.json | 27 +++++++++++++++++++++++++++ package.json | 1 + src/dev-cli.ts | 2 +- tsconfig.json | 5 ++++- 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6606ff1..b1f296f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34,6 +34,7 @@ "spinnies": "^0.5.1", "ts-node": "^10.9.1", "tsc-alias": "^1.6.7", + "tsconfig-paths": "^4.2.0", "typescript": "^4.6.3", "vite": "^4.0.0", "vite-plugin-dts": "^3.5.3", @@ -4416,6 +4417,18 @@ "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", "dev": true }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/jsonc-parser": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.2.0.tgz", @@ -5994,6 +6007,20 @@ "node": "^12.20.0 || >=14" } }, + "node_modules/tsconfig-paths": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-4.2.0.tgz", + "integrity": "sha512-NoZ4roiN7LnbKn9QqE1amc9DJfzvZXxF4xDavcOWt1BPkdx+m+0gJuPM+S0vCe7zTJMYUP0R8pO2XMr+Y8oLIg==", + "dev": true, + "dependencies": { + "json5": "^2.2.2", + "minimist": "^1.2.6", + "strip-bom": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/tslib": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", diff --git a/package.json b/package.json index d46ab74..c0cc9b4 100644 --- a/package.json +++ b/package.json @@ -65,6 +65,7 @@ "spinnies": "^0.5.1", "ts-node": "^10.9.1", "tsc-alias": "^1.6.7", + "tsconfig-paths": "^4.2.0", "typescript": "^4.6.3", "vite": "^4.0.0", "vite-plugin-dts": "^3.5.3", diff --git a/src/dev-cli.ts b/src/dev-cli.ts index 78cd4dd..16e4949 100644 --- a/src/dev-cli.ts +++ b/src/dev-cli.ts @@ -8,7 +8,7 @@ import nodeFetch from 'node-fetch'; // eslint-disable-next-line import/no-extraneous-dependencies import Spinnies from 'spinnies'; -import { MetaOutput, MovieMedia, ProviderControls, ShowMedia, makeProviders, makeStandardFetcher, targets } from '..'; +import { MetaOutput, MovieMedia, ProviderControls, ShowMedia, makeProviders, makeStandardFetcher, targets } from '.'; dotenv.config(); diff --git a/tsconfig.json b/tsconfig.json index 3c5f515..a57441a 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -17,5 +17,8 @@ } }, "include": ["src"], - "exclude": ["node_modules", "**/__tests__/*"] + "exclude": ["node_modules", "**/__tests__/*"], + "ts-node": { + "require": ["tsconfig-paths/register"] + } } From e8235072eace2d11db385a53209d41c56687a9fe Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Tue, 26 Sep 2023 16:08:42 -0400 Subject: [PATCH 17/17] appease the linting gods --- src/dev-cli.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/dev-cli.ts b/src/dev-cli.ts index 16e4949..2e691da 100644 --- a/src/dev-cli.ts +++ b/src/dev-cli.ts @@ -332,7 +332,11 @@ async function runQuestions() { options.fetcher = answers.fetcher; options.sourceId = answers.source; - const source = sources.find(({ id }) => id === answers.source)!; + const source = sources.find(({ id }) => id === answers.source); + + if (!source) { + throw new Error(`No source with ID ${answers.source} found`); + } if (source.type === 'embed') { const sourceAnswers = await prompt([