From 2a2e4a4178ae459b2932ae99fa87f8ac12fd5a38 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 21 Jun 2023 15:39:20 +0200 Subject: [PATCH 01/23] remove tslint --- package.json | 2 -- tslint.json | 3 --- 2 files changed, 5 deletions(-) delete mode 100644 tslint.json diff --git a/package.json b/package.json index e4fd977..0a369ed 100644 --- a/package.json +++ b/package.json @@ -48,8 +48,6 @@ "eslint-plugin-prettier": "^4.2.1", "prettier": "^2.6.2", "tsc-alias": "^1.6.7", - "tslint": "^6.1.3", - "tslint-config-prettier": "^1.18.0", "typescript": "^4.6.3", "vite": "^4.0.0", "vite-plugin-dts": "^2.3.0", diff --git a/tslint.json b/tslint.json deleted file mode 100644 index 70fdbab..0000000 --- a/tslint.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "extends": ["tslint:recommended", "tslint-config-prettier"] -} \ No newline at end of file From 80f7cbce0c1296a1f5646d1f284f805a8ee25c11 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 23 Aug 2023 17:56:03 +0200 Subject: [PATCH 02/23] Add repo meta files --- .github/CODEOWNERS | 3 +++ .github/CODE_OF_CONDUCT.md | 1 + .github/CONTRIBUTING.md | 1 + .github/SECURITY.md | 14 ++++++++++++++ .github/pull_request_template.md | 6 ++++++ 5 files changed, 25 insertions(+) create mode 100644 .github/CODEOWNERS create mode 100644 .github/CODE_OF_CONDUCT.md create mode 100644 .github/CONTRIBUTING.md create mode 100644 .github/SECURITY.md create mode 100644 .github/pull_request_template.md diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..d0f0ca6 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,3 @@ +* @movie-web/core + +.github @binaryoverload diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..c703492 --- /dev/null +++ b/.github/CODE_OF_CONDUCT.md @@ -0,0 +1 @@ +Please visit the [main document at primary repository](https://github.com/movie-web/movie-web/blob/dev/.github/CODE_OF_CONDUCT.md). diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 0000000..afaa8fc --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1 @@ +Please visit the [main document at primary repository](https://github.com/movie-web/movie-web/blob/dev/.github/CONTRIBUTING.md). diff --git a/.github/SECURITY.md b/.github/SECURITY.md new file mode 100644 index 0000000..c8ee568 --- /dev/null +++ b/.github/SECURITY.md @@ -0,0 +1,14 @@ +# Security Policy + +## Supported Versions + +The movie-web maintainers only support the latest version of movie-web published at https://movie-web.app. +This published version is equivalent to the master branch. + +Support is not provided for any forks or mirrors of movie-web. + +## Reporting a Vulnerability + +There are two ways you can contact the movie-web maintainers to report a vulnerability: + - Email [security@movie-web.app](mailto:security@movie-web.app) + - Report the vulnerability in the [movie-web Discord server](https://discord.movie-web.app) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..c072b7f --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,6 @@ +This pull request resolves #XXX + + - [ ] I have read and agreed to the [code of conduct](https://github.com/movie-web/movie-web/blob/dev/.github/CODE_OF_CONDUCT.md). + - [ ] I have read and complied with the [contributing guidelines](https://github.com/movie-web/movie-web/blob/dev/.github/CONTRIBUTING.md). + - [ ] What I'm implementing was assigned to me and is an [approved issue](https://github.com/movie-web/movie-web/issues?q=is%3Aopen+is%3Aissue+label%3Aapproved). For reference, please take a look at our [GitHub projects](https://github.com/movie-web/movie-web/projects). + - [ ] I have tested all of my changes. From 61ee0e13fa0b56e78576319d32adcdc3be3d7598 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 23 Aug 2023 17:56:18 +0200 Subject: [PATCH 03/23] Start building a provider scrape system --- src/index.ts | 6 +----- src/main/builder.ts | 16 ++++++++++++++++ src/providers/all.ts | 3 +++ src/providers/base.ts | 14 ++++++++++++++ src/providers/embeds/.gitkeep | 0 src/providers/sources/.gitkeep | 0 src/providers/streams.ts | 18 ++++++++++++++++++ src/testing/oof.ts | 1 - src/utils/context.ts | 6 ++++++ src/utils/fetcher.ts | 11 +++++++++++ 10 files changed, 69 insertions(+), 6 deletions(-) create mode 100644 src/main/builder.ts create mode 100644 src/providers/all.ts create mode 100644 src/providers/base.ts create mode 100644 src/providers/embeds/.gitkeep create mode 100644 src/providers/sources/.gitkeep create mode 100644 src/providers/streams.ts delete mode 100644 src/testing/oof.ts create mode 100644 src/utils/context.ts create mode 100644 src/utils/fetcher.ts diff --git a/src/index.ts b/src/index.ts index 6cce377..89e547d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1 @@ -import { LOG } from '@/testing/oof'; - -export function test() { - console.log(LOG); -} +export { ProviderBuilderOptions, Providers, makeProviders } from '@/main/builder'; diff --git a/src/main/builder.ts b/src/main/builder.ts new file mode 100644 index 0000000..8033ca7 --- /dev/null +++ b/src/main/builder.ts @@ -0,0 +1,16 @@ +import { Fetcher } from '@/utils/fetcher'; + +export interface ProviderBuilderOptions { + // fetcher, every web request gets called through here + fetcher: Fetcher; + + // proxied fetcher, if the scraper needs to access a CORS proxy. this fetcher will be called instead + // of the normal fetcher. Defaults to the normal fetcher. + proxiedFetcher?: Fetcher; +} + +export interface Providers {} + +export function makeProviders(ops: ProviderBuilderOptions): Providers { + return {}; +} diff --git a/src/providers/all.ts b/src/providers/all.ts new file mode 100644 index 0000000..eb474bb --- /dev/null +++ b/src/providers/all.ts @@ -0,0 +1,3 @@ +import { Sourcerer } from '@/providers/base'; + +export const sources: Array = []; diff --git a/src/providers/base.ts b/src/providers/base.ts new file mode 100644 index 0000000..210ce3a --- /dev/null +++ b/src/providers/base.ts @@ -0,0 +1,14 @@ +import { ScrapeContext } from '@/utils/context'; + +export type Sourcerer = { + id: string; + name: string; // displayed in the UI + rank: number; // the higher the number, the earlier it gets put on the queue + disabled?: boolean; + scrape: (input: ScrapeContext) => void; +}; + +export function makeSourcerer(state: Sourcerer): Sourcerer | null { + if (state.disabled) return null; + return state; +} diff --git a/src/providers/embeds/.gitkeep b/src/providers/embeds/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/src/providers/sources/.gitkeep b/src/providers/sources/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/src/providers/streams.ts b/src/providers/streams.ts new file mode 100644 index 0000000..6078628 --- /dev/null +++ b/src/providers/streams.ts @@ -0,0 +1,18 @@ +export type StreamFile = { + type: 'mp4'; + url: string; +}; + +export type Qualities = '360' | '480' | '720' | '1080'; + +export type FileBasedStream = { + type: 'file'; + qualities: Record; +}; + +export type HlsBasedStream = { + type: 'hls'; + playlist: string; +}; + +export type Stream = FileBasedStream | HlsBasedStream; diff --git a/src/testing/oof.ts b/src/testing/oof.ts deleted file mode 100644 index 5cfd8a6..0000000 --- a/src/testing/oof.ts +++ /dev/null @@ -1 +0,0 @@ -export const LOG = 'hello world'; diff --git a/src/utils/context.ts b/src/utils/context.ts new file mode 100644 index 0000000..4bbe37d --- /dev/null +++ b/src/utils/context.ts @@ -0,0 +1,6 @@ +import { Fetcher } from '@/utils/fetcher'; + +export interface ScrapeContext { + proxiedFetcher: Fetcher; + fetcher: Fetcher; +} diff --git a/src/utils/fetcher.ts b/src/utils/fetcher.ts new file mode 100644 index 0000000..83dd608 --- /dev/null +++ b/src/utils/fetcher.ts @@ -0,0 +1,11 @@ +export type FetcherOptions = { + baseUrl?: string; + headers?: Record; + query?: Record; + method?: 'GET' | 'POST'; + body?: Record | string | FormData; +}; + +export type Fetcher = { + (url: string, ops?: FetcherOptions): T; +}; From fe721bee37b714f8b9758aa5fdccc4919f64e4f2 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 23 Aug 2023 17:59:10 +0200 Subject: [PATCH 04/23] example comments --- src/main/builder.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/builder.ts b/src/main/builder.ts index 8033ca7..65a6c26 100644 --- a/src/main/builder.ts +++ b/src/main/builder.ts @@ -14,3 +14,13 @@ export interface Providers {} export function makeProviders(ops: ProviderBuilderOptions): Providers { return {}; } + +// + +// const scrapers = makeProviders({ +// fetcher: makeFetcher(fetch), +// }); + +// scrapers.scrape(); +// scrapers.callScraper(id); +// scrapers.getScraper(id); From ef766936dd97c4f9b1fff32eb24aede68a933da0 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 23 Aug 2023 20:07:39 +0200 Subject: [PATCH 05/23] Basically library structure --- src/fetchers/common.ts | 15 ++++++++++ src/fetchers/standardFetch.ts | 15 ++++++++++ src/fetchers/types.ts | 24 ++++++++++++++++ src/index.ts | 2 +- src/main/builder.ts | 54 +++++++++++++++++++++++++++-------- src/main/meta.ts | 37 ++++++++++++++++++++++++ src/main/runner.ts | 18 ++++++++++++ src/providers/all.ts | 23 ++++++++++++++- src/providers/base.ts | 7 ++++- src/utils/context.ts | 7 +++-- src/utils/fetcher.ts | 11 ------- src/utils/predicates.ts | 7 +++++ 12 files changed, 191 insertions(+), 29 deletions(-) create mode 100644 src/fetchers/common.ts create mode 100644 src/fetchers/standardFetch.ts create mode 100644 src/fetchers/types.ts create mode 100644 src/main/meta.ts create mode 100644 src/main/runner.ts delete mode 100644 src/utils/fetcher.ts create mode 100644 src/utils/predicates.ts diff --git a/src/fetchers/common.ts b/src/fetchers/common.ts new file mode 100644 index 0000000..6ce7c87 --- /dev/null +++ b/src/fetchers/common.ts @@ -0,0 +1,15 @@ +import { FetcherOptions } from '@/fetchers/types'; + +// make url with query params and base url used correctly +export function makeFullUrl(url: string, ops?: FetcherOptions): string { + // glue baseUrl and rest of url together + const fullUrl = ops?.baseUrl ?? ''; + // TODO make full url + + const parsedUrl = new URL(fullUrl); + Object.entries(ops?.query ?? {}).forEach(([k, v]) => { + parsedUrl.searchParams.set(k, v); + }); + + return parsedUrl.toString(); +} diff --git a/src/fetchers/standardFetch.ts b/src/fetchers/standardFetch.ts new file mode 100644 index 0000000..54ad91e --- /dev/null +++ b/src/fetchers/standardFetch.ts @@ -0,0 +1,15 @@ +import { makeFullUrl } from '@/fetchers/common'; +import { Fetcher } from '@/fetchers/types'; + +export function makeStandardFetcher(f: typeof fetch): Fetcher { + const normalFetch: Fetcher = (url, ops) => { + const fullUrl = makeFullUrl(url, ops); + + return f(fullUrl, { + method: ops.method, + body: JSON.stringify(ops.body), // TODO content type headers + proper serialization + }); + }; + + return normalFetch; +} diff --git a/src/fetchers/types.ts b/src/fetchers/types.ts new file mode 100644 index 0000000..62f78b9 --- /dev/null +++ b/src/fetchers/types.ts @@ -0,0 +1,24 @@ +export type FetcherOptions = { + baseUrl?: string; + headers?: Record; + query?: Record; + method?: 'GET' | 'POST'; + body?: Record | string | FormData; +}; + +export type DefaultedFetcherOptions = { + baseUrl?: string; + body?: Record | string | FormData; + headers: Record; + query: Record; + method: 'GET' | 'POST'; +}; + +export type Fetcher = { + (url: string, ops: DefaultedFetcherOptions): T; +}; + +// this feature has some quality of life features +export type UseableFetcher = { + (url: string, ops?: FetcherOptions): T; +}; diff --git a/src/index.ts b/src/index.ts index 89e547d..97173d5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1 +1 @@ -export { ProviderBuilderOptions, Providers, makeProviders } from '@/main/builder'; +export { ProviderBuilderOptions, ProviderControls, makeProviders } from '@/main/builder'; diff --git a/src/main/builder.ts b/src/main/builder.ts index 65a6c26..7d594d0 100644 --- a/src/main/builder.ts +++ b/src/main/builder.ts @@ -1,4 +1,13 @@ -import { Fetcher } from '@/utils/fetcher'; +import { Fetcher } from '@/fetchers/types'; +import { MetaOutput, getAllEmbedMetaSorted, getAllSourceMetaSorted, getSpecificId } from '@/main/meta'; +import { EmbedRunOutput, RunOutput, SourceRunOutput } from '@/main/runner'; +import { getProviders } from '@/providers/all'; + +// TODO meta data input (tmdb id, imdb id, title, release year) +// TODO actually running scrapers +// TODO documentation: examples for nodejs + browser +// TODO documentation: how to use + usecases +// TODO documentation: examples on how to make a custom fetcher export interface ProviderBuilderOptions { // fetcher, every web request gets called through here @@ -9,18 +18,39 @@ export interface ProviderBuilderOptions { proxiedFetcher?: Fetcher; } -export interface Providers {} +export interface ProviderControls { + // Run all providers one by one. in order of rank (highest first) + // returns the stream, or null if none found + runAll(): Promise; -export function makeProviders(ops: ProviderBuilderOptions): Providers { - return {}; + // Run a source provider + runSource(id: string): Promise; + + // Run a embed provider + runEmbed(id: string): Promise; + + // get meta data about a source or embed. + getMetadata(id: string): MetaOutput | null; + + // return all sources. sorted by rank (highest first) + listSources(): MetaOutput[]; + + // return all embed scrapers. sorted by rank (highest first) + listEmbeds(): MetaOutput[]; } -// +export function makeProviders(_ops: ProviderBuilderOptions): ProviderControls { + const list = getProviders(); -// const scrapers = makeProviders({ -// fetcher: makeFetcher(fetch), -// }); - -// scrapers.scrape(); -// scrapers.callScraper(id); -// scrapers.getScraper(id); + return { + getMetadata(id) { + return getSpecificId(list, id); + }, + listSources() { + return getAllSourceMetaSorted(list); + }, + listEmbeds() { + return getAllEmbedMetaSorted(list); + }, + }; +} diff --git a/src/main/meta.ts b/src/main/meta.ts new file mode 100644 index 0000000..d2a05b4 --- /dev/null +++ b/src/main/meta.ts @@ -0,0 +1,37 @@ +import { ProviderList } from '@/providers/all'; + +export type MetaOutput = { + type: 'embed' | 'source'; + id: string; + rank: number; + name: string; +}; + +export function getAllSourceMetaSorted(list: ProviderList): MetaOutput[] { + return list.sources + .sort((a, b) => b.rank - a.rank) + .map((v) => ({ + type: 'source', + id: v.id, + name: v.name, + rank: v.rank, + })); +} + +export function getAllEmbedMetaSorted(_list: ProviderList): MetaOutput[] { + return []; +} + +export function getSpecificId(list: ProviderList, id: string): MetaOutput | null { + const foundSource = list.sources.find((v) => v.id === id); + if (foundSource) { + return { + type: 'source', + id: foundSource.id, + name: foundSource.name, + rank: foundSource.rank, + }; + } + + return null; +} diff --git a/src/main/runner.ts b/src/main/runner.ts new file mode 100644 index 0000000..f0295af --- /dev/null +++ b/src/main/runner.ts @@ -0,0 +1,18 @@ +import { Stream } from '@/providers/streams'; + +export type RunOutput = { + sourceId: string; + fromEmbed: boolean; + stream: Stream; +}; + +export type SourceRunOutput = { + sourceId: string; + stream?: Stream; + embeds: []; +}; + +export type EmbedRunOutput = { + embedId: string; + stream?: Stream; +}; diff --git a/src/providers/all.ts b/src/providers/all.ts index eb474bb..9ad3156 100644 --- a/src/providers/all.ts +++ b/src/providers/all.ts @@ -1,3 +1,24 @@ import { Sourcerer } from '@/providers/base'; +import { hasDuplicates, isNotNull } from '@/utils/predicates'; -export const sources: Array = []; +function gatherAllSources(): Array { + return []; +} + +export interface ProviderList { + sources: Sourcerer[]; +} + +export function getProviders(): ProviderList { + const sources = gatherAllSources().filter(isNotNull); + + const anyDuplicateId = hasDuplicates(sources.map((v) => v.id)); + const anyDuplicateRank = hasDuplicates(sources.map((v) => v.rank)); + + if (anyDuplicateId) throw new Error('Duplicate id found in sources'); + if (anyDuplicateRank) throw new Error('Duplicate rank found in sources'); + + return { + sources, + }; +} diff --git a/src/providers/base.ts b/src/providers/base.ts index 210ce3a..8292690 100644 --- a/src/providers/base.ts +++ b/src/providers/base.ts @@ -1,11 +1,16 @@ +import { Stream } from '@/providers/streams'; import { ScrapeContext } from '@/utils/context'; +export type SourcererOutput = { + stream?: Stream; +}; + export type Sourcerer = { id: string; name: string; // displayed in the UI rank: number; // the higher the number, the earlier it gets put on the queue disabled?: boolean; - scrape: (input: ScrapeContext) => void; + scrape: (input: ScrapeContext) => Promise; }; export function makeSourcerer(state: Sourcerer): Sourcerer | null { diff --git a/src/utils/context.ts b/src/utils/context.ts index 4bbe37d..3e42ffa 100644 --- a/src/utils/context.ts +++ b/src/utils/context.ts @@ -1,6 +1,7 @@ -import { Fetcher } from '@/utils/fetcher'; +import { UseableFetcher } from '@/fetchers/types'; export interface ScrapeContext { - proxiedFetcher: Fetcher; - fetcher: Fetcher; + proxiedFetcher: UseableFetcher; + fetcher: UseableFetcher; + progress(val: number): void; } diff --git a/src/utils/fetcher.ts b/src/utils/fetcher.ts deleted file mode 100644 index 83dd608..0000000 --- a/src/utils/fetcher.ts +++ /dev/null @@ -1,11 +0,0 @@ -export type FetcherOptions = { - baseUrl?: string; - headers?: Record; - query?: Record; - method?: 'GET' | 'POST'; - body?: Record | string | FormData; -}; - -export type Fetcher = { - (url: string, ops?: FetcherOptions): T; -}; diff --git a/src/utils/predicates.ts b/src/utils/predicates.ts new file mode 100644 index 0000000..03e7469 --- /dev/null +++ b/src/utils/predicates.ts @@ -0,0 +1,7 @@ +export function isNotNull(value: T | null | undefined): value is T { + return value !== null && value !== undefined; +} + +export function hasDuplicates(values: Array): boolean { + return new Set(values).size !== values.length; +} From ffe5cf536904d5a6ee3d6b210a887265ed75912e Mon Sep 17 00:00:00 2001 From: mrjvs Date: Thu, 24 Aug 2023 21:51:43 +0200 Subject: [PATCH 06/23] callback events --- README.md | 7 ++++++ src/main/builder.ts | 22 ++++++++++--------- src/main/events.ts | 46 ++++++++++++++++++++++++++++++++++++++++ src/main/runner.ts | 19 ++++++++++++++++- src/providers/base.ts | 23 +++++++++++++++++++- src/providers/streams.ts | 2 +- src/utils/context.ts | 10 +++++++-- src/utils/errors.ts | 6 ++++++ 8 files changed, 120 insertions(+), 15 deletions(-) create mode 100644 src/main/events.ts create mode 100644 src/utils/errors.ts diff --git a/README.md b/README.md index e69ac12..14a1337 100644 --- a/README.md +++ b/README.md @@ -6,5 +6,12 @@ Feel free to use for your own projects. features: - scrape popular streaming websites - works in both browser and NodeJS server + - choose between all streams or non-protected stream (for browser use) > This package is still WIP + +> TODO documentation: examples for nodejs + browser + +> TODO documentation: how to use + usecases + +> TODO documentation: examples on how to make a custom fetcher diff --git a/src/main/builder.ts b/src/main/builder.ts index 7d594d0..ee96ba1 100644 --- a/src/main/builder.ts +++ b/src/main/builder.ts @@ -1,13 +1,11 @@ import { Fetcher } from '@/fetchers/types'; +import { FullScraperEvents, SingleScraperEvents } from '@/main/events'; import { MetaOutput, getAllEmbedMetaSorted, getAllSourceMetaSorted, getSpecificId } from '@/main/meta'; -import { EmbedRunOutput, RunOutput, SourceRunOutput } from '@/main/runner'; +import { ProviderRunnerOptions, RunOutput, SourceRunOutput, runAllProviders } from '@/main/runner'; import { getProviders } from '@/providers/all'; // TODO meta data input (tmdb id, imdb id, title, release year) // TODO actually running scrapers -// TODO documentation: examples for nodejs + browser -// TODO documentation: how to use + usecases -// TODO documentation: examples on how to make a custom fetcher export interface ProviderBuilderOptions { // fetcher, every web request gets called through here @@ -21,13 +19,10 @@ export interface ProviderBuilderOptions { export interface ProviderControls { // Run all providers one by one. in order of rank (highest first) // returns the stream, or null if none found - runAll(): Promise; + runAll(cbs: FullScraperEvents): Promise; // Run a source provider - runSource(id: string): Promise; - - // Run a embed provider - runEmbed(id: string): Promise; + runSource(id: string, cbs: SingleScraperEvents): Promise; // get meta data about a source or embed. getMetadata(id: string): MetaOutput | null; @@ -39,10 +34,17 @@ export interface ProviderControls { listEmbeds(): MetaOutput[]; } -export function makeProviders(_ops: ProviderBuilderOptions): ProviderControls { +export function makeProviders(ops: ProviderBuilderOptions): ProviderControls { const list = getProviders(); + const providerRunnerOps: ProviderRunnerOptions = { + fetcher: ops.fetcher, + proxiedFetcher: ops.proxiedFetcher ?? ops.fetcher, + }; return { + runAll(cbs) { + return runAllProviders(providerRunnerOps, cbs); + }, getMetadata(id) { return getSpecificId(list, id); }, diff --git a/src/main/events.ts b/src/main/events.ts new file mode 100644 index 0000000..09f648d --- /dev/null +++ b/src/main/events.ts @@ -0,0 +1,46 @@ +export type UpdateEventStatus = 'success' | 'failure' | 'notfound' | 'pending'; + +export type UpdateEvent = { + percentage: number; + status: UpdateEventStatus; +}; + +export type InitEvent = { + sourceIds: string[]; // list of source ids +}; + +export type DiscoverEmbedsEvent = { + sourceId: string; + + // list of embeds that will be scraped in order + embeds: Array<{ + id: string; + embedScraperId: string; + }>; +}; + +export type StartScrapingEvent = { + sourceId: string; + + // embed Id (not embedScraperId) + embedId?: string; +}; + +export type SingleScraperEvents = { + update?: (evt: UpdateEvent) => void; +}; + +export type FullScraperEvents = { + // update progress percentage and status of the currently scraping item + update?: (evt: UpdateEvent) => void; + + // initial list of scrapers its running, only triggers once per run. + init?: (evt: InitEvent) => void; + + // list of embeds are discovered for the currently running source scraper + // triggers once per source scraper + discoverEmbeds?: (evt: DiscoverEmbedsEvent) => void; + + // start scraping an item. + start?: (id: string) => void; +}; diff --git a/src/main/runner.ts b/src/main/runner.ts index f0295af..b2627e0 100644 --- a/src/main/runner.ts +++ b/src/main/runner.ts @@ -1,8 +1,10 @@ +import { Fetcher } from '@/fetchers/types'; +import { FullScraperEvents } from '@/main/events'; import { Stream } from '@/providers/streams'; export type RunOutput = { sourceId: string; - fromEmbed: boolean; + embedId?: string; stream: Stream; }; @@ -16,3 +18,18 @@ export type EmbedRunOutput = { embedId: string; stream?: Stream; }; + +export type ProviderRunnerOptions = { + fetcher: Fetcher; + proxiedFetcher: Fetcher; +}; + +export async function runAllProviders(_ops: ProviderRunnerOptions, _cbs: FullScraperEvents): Promise { + return { + sourceId: '123', + stream: { + type: 'file', + qualities: {}, + }, + }; +} diff --git a/src/providers/base.ts b/src/providers/base.ts index 8292690..373989a 100644 --- a/src/providers/base.ts +++ b/src/providers/base.ts @@ -1,7 +1,11 @@ import { Stream } from '@/providers/streams'; -import { ScrapeContext } from '@/utils/context'; +import { EmbedScrapeContext, ScrapeContext } from '@/utils/context'; export type SourcererOutput = { + embeds: { + embedId: string; + url: string; + }[]; stream?: Stream; }; @@ -17,3 +21,20 @@ export function makeSourcerer(state: Sourcerer): Sourcerer | null { if (state.disabled) return null; return state; } + +export type EmbedOutput = { + stream?: Stream; +}; + +export type Embed = { + id: string; + name: string; // displayed in the UI + rank: number; // the higher the number, the earlier it gets put on the queue + disabled?: boolean; + scrape: (input: EmbedScrapeContext) => Promise; +}; + +export function makeEmbed(state: Embed): Embed | null { + if (state.disabled) return null; + return state; +} diff --git a/src/providers/streams.ts b/src/providers/streams.ts index 6078628..3c46567 100644 --- a/src/providers/streams.ts +++ b/src/providers/streams.ts @@ -7,7 +7,7 @@ export type Qualities = '360' | '480' | '720' | '1080'; export type FileBasedStream = { type: 'file'; - qualities: Record; + qualities: Partial>; }; export type HlsBasedStream = { diff --git a/src/utils/context.ts b/src/utils/context.ts index 3e42ffa..5d89f96 100644 --- a/src/utils/context.ts +++ b/src/utils/context.ts @@ -1,7 +1,13 @@ import { UseableFetcher } from '@/fetchers/types'; -export interface ScrapeContext { +export type ScrapeContext = { proxiedFetcher: UseableFetcher; fetcher: UseableFetcher; progress(val: number): void; -} +}; + +export type EmbedInput = { + url: string; +}; + +export type EmbedScrapeContext = EmbedInput & ScrapeContext; diff --git a/src/utils/errors.ts b/src/utils/errors.ts new file mode 100644 index 0000000..d31f7d8 --- /dev/null +++ b/src/utils/errors.ts @@ -0,0 +1,6 @@ +export class NotFoundError extends Error { + constructor(reason?: string) { + super(`Couldn't found a stream: ${reason ?? 'not found'}`); + this.name = 'NotFoundError'; + } +} From 59b2aa22f7bbef0f4e0deac503ff9f9f2dbd5ea0 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Fri, 25 Aug 2023 22:29:50 +0200 Subject: [PATCH 07/23] running sources/embeds + media input --- README.md | 12 +++++++++++ docs/basic.js | 19 +++++++++++++++++ src/index.ts | 1 + src/main/builder.ts | 36 ++++++++++++++++++++++----------- src/main/media.ts | 26 ++++++++++++++++++++++++ src/main/runner.ts | 7 ++++++- src/providers/base.ts | 4 +++- src/providers/sources/flixhq.ts | 28 +++++++++++++++++++++++++ 8 files changed, 119 insertions(+), 14 deletions(-) create mode 100644 docs/basic.js create mode 100644 src/main/media.ts create mode 100644 src/providers/sources/flixhq.ts diff --git a/README.md b/README.md index 14a1337..f9b5661 100644 --- a/README.md +++ b/README.md @@ -15,3 +15,15 @@ features: > TODO documentation: how to use + usecases > TODO documentation: examples on how to make a custom fetcher + +> TODO functionality: running individual scrapers + +> TODO functionality: running all scrapers + +> TODO functionality: choose environment (for browser, for native) + +> TODO functionality: show which types are supported for scraper in meta + +> TODO content: add all scrapers/providers + +> TODO tests: add tests diff --git a/docs/basic.js b/docs/basic.js new file mode 100644 index 0000000..7acc249 --- /dev/null +++ b/docs/basic.js @@ -0,0 +1,19 @@ +async function example() { + const providers = makeProviders({ + fetcher: makeStandardFetcher(fetch), + }); + + const source = await providers.runAll({ + media: { + title: 'Spider-Man: Across the Spider-Verse', + releaseYear: 2023, + imbdId: 'tt9362722', + tmdbId: '569094', + type: 'movie', + }, + }); + + if (!source) throw new Error("Couldn't find a stream"); + if (source.stream.type === 'file') return source.stream.qualities['1080']?.url; + if (source.stream.type === 'hls') return source.stream.playlist; +} diff --git a/src/index.ts b/src/index.ts index 97173d5..9025534 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1 +1,2 @@ +export { makeStandardFetcher } from '@/fetchers/standardFetch'; export { ProviderBuilderOptions, ProviderControls, makeProviders } from '@/main/builder'; diff --git a/src/main/builder.ts b/src/main/builder.ts index ee96ba1..bb4d114 100644 --- a/src/main/builder.ts +++ b/src/main/builder.ts @@ -1,12 +1,10 @@ import { Fetcher } from '@/fetchers/types'; -import { FullScraperEvents, SingleScraperEvents } from '@/main/events'; +import { FullScraperEvents } from '@/main/events'; +import { ScrapeMedia } from '@/main/media'; import { MetaOutput, getAllEmbedMetaSorted, getAllSourceMetaSorted, getSpecificId } from '@/main/meta'; -import { ProviderRunnerOptions, RunOutput, SourceRunOutput, runAllProviders } from '@/main/runner'; +import { RunOutput, runAllProviders } from '@/main/runner'; import { getProviders } from '@/providers/all'; -// TODO meta data input (tmdb id, imdb id, title, release year) -// TODO actually running scrapers - export interface ProviderBuilderOptions { // fetcher, every web request gets called through here fetcher: Fetcher; @@ -16,13 +14,24 @@ export interface ProviderBuilderOptions { proxiedFetcher?: Fetcher; } +export interface RunnerOptions { + // overwrite the order of sources to run. list of ids + sourceOrder?: string[]; + + // overwrite the order of embeds to run. list of ids + embedOrder?: string[]; + + // object of event functions + events?: FullScraperEvents; + + // the media you want to see sources from + media: ScrapeMedia; +} + export interface ProviderControls { // Run all providers one by one. in order of rank (highest first) // returns the stream, or null if none found - runAll(cbs: FullScraperEvents): Promise; - - // Run a source provider - runSource(id: string, cbs: SingleScraperEvents): Promise; + runAll(runnerOps: RunnerOptions): Promise; // get meta data about a source or embed. getMetadata(id: string): MetaOutput | null; @@ -36,14 +45,17 @@ export interface ProviderControls { export function makeProviders(ops: ProviderBuilderOptions): ProviderControls { const list = getProviders(); - const providerRunnerOps: ProviderRunnerOptions = { + const providerRunnerOps = { fetcher: ops.fetcher, proxiedFetcher: ops.proxiedFetcher ?? ops.fetcher, }; return { - runAll(cbs) { - return runAllProviders(providerRunnerOps, cbs); + runAll(runnerOps: RunnerOptions) { + return runAllProviders({ + ...providerRunnerOps, + ...runnerOps, + }); }, getMetadata(id) { return getSpecificId(list, id); diff --git a/src/main/media.ts b/src/main/media.ts new file mode 100644 index 0000000..8acc8e1 --- /dev/null +++ b/src/main/media.ts @@ -0,0 +1,26 @@ +export type CommonMedia = { + title: string; + releaseYear: number; + imbdId: string; + tmdbId: string; +}; + +export type MediaTypes = 'show' | 'movie'; + +export type ShowMedia = CommonMedia & { + type: 'show'; + episode: { + number: number; + tmdbId: string; + }; + season: { + number: number; + tmdbId: string; + }; +}; + +export type MovieMedia = CommonMedia & { + type: 'movie'; +}; + +export type ScrapeMedia = ShowMedia | MovieMedia; diff --git a/src/main/runner.ts b/src/main/runner.ts index b2627e0..75203af 100644 --- a/src/main/runner.ts +++ b/src/main/runner.ts @@ -1,5 +1,6 @@ import { Fetcher } from '@/fetchers/types'; import { FullScraperEvents } from '@/main/events'; +import { ScrapeMedia } from '@/main/media'; import { Stream } from '@/providers/streams'; export type RunOutput = { @@ -22,9 +23,13 @@ export type EmbedRunOutput = { export type ProviderRunnerOptions = { fetcher: Fetcher; proxiedFetcher: Fetcher; + sourceOrder?: string[]; + embedOrder?: string[]; + events?: FullScraperEvents; + media: ScrapeMedia; }; -export async function runAllProviders(_ops: ProviderRunnerOptions, _cbs: FullScraperEvents): Promise { +export async function runAllProviders(_ops: ProviderRunnerOptions): Promise { return { sourceId: '123', stream: { diff --git a/src/providers/base.ts b/src/providers/base.ts index 373989a..df414f2 100644 --- a/src/providers/base.ts +++ b/src/providers/base.ts @@ -1,3 +1,4 @@ +import { MovieMedia, ShowMedia } from '@/main/media'; import { Stream } from '@/providers/streams'; import { EmbedScrapeContext, ScrapeContext } from '@/utils/context'; @@ -14,7 +15,8 @@ export type Sourcerer = { name: string; // displayed in the UI rank: number; // the higher the number, the earlier it gets put on the queue disabled?: boolean; - scrape: (input: ScrapeContext) => Promise; + scrapeMovie?: (input: ScrapeContext & { media: MovieMedia }) => Promise; + scrapeShow: (input: ScrapeContext & { media: ShowMedia }) => Promise; }; export function makeSourcerer(state: Sourcerer): Sourcerer | null { diff --git a/src/providers/sources/flixhq.ts b/src/providers/sources/flixhq.ts new file mode 100644 index 0000000..eff0750 --- /dev/null +++ b/src/providers/sources/flixhq.ts @@ -0,0 +1,28 @@ +import { makeSourcerer } from '@/providers/base'; + +export const flixHq = makeSourcerer({ + id: 'flixhq', + name: 'FlixHQ', + rank: 500, + + async scrapeShow(_input) { + return { + embeds: [], + stream: { + type: 'file', + qualities: { + '360': { + type: 'mp4', + url: 'blabal.mp4', + }, + }, + }, + }; + }, + + async scrapeMovie(_input) { + return { + embeds: [], + }; + }, +}); From c55f830c307a86de7da26f049d0f312d9d48c4ef Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sun, 27 Aug 2023 19:47:09 +0200 Subject: [PATCH 08/23] Add runner --- README.md | 7 +-- src/fetchers/common.ts | 14 ++++- src/index.ts | 9 ++- src/main/builder.ts | 9 ++- src/main/meta.ts | 20 ++++-- src/main/runner.ts | 135 ++++++++++++++++++++++++++++++++++++++--- src/providers/all.ts | 22 +++++-- src/providers/base.ts | 4 +- src/utils/list.ts | 20 ++++++ 9 files changed, 207 insertions(+), 33 deletions(-) create mode 100644 src/utils/list.ts diff --git a/README.md b/README.md index f9b5661..b5fa78a 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,7 @@ Feel free to use for your own projects. features: - scrape popular streaming websites - - works in both browser and NodeJS server - - choose between all streams or non-protected stream (for browser use) + - works in both browser and server-side > This package is still WIP @@ -18,12 +17,8 @@ features: > TODO functionality: running individual scrapers -> TODO functionality: running all scrapers - > TODO functionality: choose environment (for browser, for native) -> TODO functionality: show which types are supported for scraper in meta - > TODO content: add all scrapers/providers > TODO tests: add tests diff --git a/src/fetchers/common.ts b/src/fetchers/common.ts index 6ce7c87..e0c1780 100644 --- a/src/fetchers/common.ts +++ b/src/fetchers/common.ts @@ -1,4 +1,4 @@ -import { FetcherOptions } from '@/fetchers/types'; +import { Fetcher, FetcherOptions, UseableFetcher } from '@/fetchers/types'; // make url with query params and base url used correctly export function makeFullUrl(url: string, ops?: FetcherOptions): string { @@ -13,3 +13,15 @@ export function makeFullUrl(url: string, ops?: FetcherOptions): string { return parsedUrl.toString(); } + +export function makeFullFetcher(fetcher: Fetcher): UseableFetcher { + return (url, ops) => { + return fetcher(url, { + headers: ops?.headers ?? {}, + method: ops?.method ?? 'GET', + query: ops?.query ?? {}, + baseUrl: ops?.baseUrl ?? '', + body: ops?.body, + }); + }; +} diff --git a/src/index.ts b/src/index.ts index 9025534..f6b45ed 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,2 +1,9 @@ +export type { RunOutput } from '@/main/runner'; +export type { MetaOutput } from '@/main/meta'; +export type { FullScraperEvents } from '@/main/events'; +export type { MediaTypes, ShowMedia, ScrapeMedia, MovieMedia } from '@/main/media'; +export type { ProviderBuilderOptions, ProviderControls, RunnerOptions } from '@/main/builder'; + +export { NotFoundError } from '@/utils/errors'; +export { makeProviders } from '@/main/builder'; export { makeStandardFetcher } from '@/fetchers/standardFetch'; -export { ProviderBuilderOptions, ProviderControls, makeProviders } from '@/main/builder'; diff --git a/src/main/builder.ts b/src/main/builder.ts index bb4d114..2cf50fe 100644 --- a/src/main/builder.ts +++ b/src/main/builder.ts @@ -1,3 +1,4 @@ +import { makeFullFetcher } from '@/fetchers/common'; import { Fetcher } from '@/fetchers/types'; import { FullScraperEvents } from '@/main/events'; import { ScrapeMedia } from '@/main/media'; @@ -16,9 +17,11 @@ export interface ProviderBuilderOptions { export interface RunnerOptions { // overwrite the order of sources to run. list of ids + // any omitted ids are in added to the end in order of rank (highest first) sourceOrder?: string[]; // overwrite the order of embeds to run. list of ids + // any omitted ids are in added to the end in order of rank (highest first) embedOrder?: string[]; // object of event functions @@ -46,13 +49,13 @@ export interface ProviderControls { export function makeProviders(ops: ProviderBuilderOptions): ProviderControls { const list = getProviders(); const providerRunnerOps = { - fetcher: ops.fetcher, - proxiedFetcher: ops.proxiedFetcher ?? ops.fetcher, + fetcher: makeFullFetcher(ops.fetcher), + proxiedFetcher: makeFullFetcher(ops.proxiedFetcher ?? ops.fetcher), }; return { runAll(runnerOps: RunnerOptions) { - return runAllProviders({ + return runAllProviders(list, { ...providerRunnerOps, ...runnerOps, }); diff --git a/src/main/meta.ts b/src/main/meta.ts index d2a05b4..cbd8297 100644 --- a/src/main/meta.ts +++ b/src/main/meta.ts @@ -1,3 +1,4 @@ +import { MediaTypes } from '@/main/media'; import { ProviderList } from '@/providers/all'; export type MetaOutput = { @@ -5,17 +6,24 @@ export type MetaOutput = { id: string; rank: number; name: string; + mediaTypes?: Array; }; export function getAllSourceMetaSorted(list: ProviderList): MetaOutput[] { return list.sources .sort((a, b) => b.rank - a.rank) - .map((v) => ({ - type: 'source', - id: v.id, - name: v.name, - rank: v.rank, - })); + .map((v) => { + const types: Array = []; + if (v.scrapeMovie) types.push('movie'); + if (v.scrapeShow) types.push('show'); + return { + type: 'source', + id: v.id, + rank: v.rank, + name: v.name, + mediaTypes: types, + }; + }); } export function getAllEmbedMetaSorted(_list: ProviderList): MetaOutput[] { diff --git a/src/main/runner.ts b/src/main/runner.ts index 75203af..e09ea00 100644 --- a/src/main/runner.ts +++ b/src/main/runner.ts @@ -1,7 +1,12 @@ -import { Fetcher } from '@/fetchers/types'; +import { UseableFetcher } from '@/fetchers/types'; import { FullScraperEvents } from '@/main/events'; import { ScrapeMedia } from '@/main/media'; +import { ProviderList } from '@/providers/all'; +import { EmbedOutput, SourcererOutput } from '@/providers/base'; import { Stream } from '@/providers/streams'; +import { ScrapeContext } from '@/utils/context'; +import { NotFoundError } from '@/utils/errors'; +import { reorderOnIdList } from '@/utils/list'; export type RunOutput = { sourceId: string; @@ -21,20 +26,132 @@ export type EmbedRunOutput = { }; export type ProviderRunnerOptions = { - fetcher: Fetcher; - proxiedFetcher: Fetcher; + fetcher: UseableFetcher; + proxiedFetcher: UseableFetcher; sourceOrder?: string[]; embedOrder?: string[]; events?: FullScraperEvents; media: ScrapeMedia; }; -export async function runAllProviders(_ops: ProviderRunnerOptions): Promise { - return { - sourceId: '123', - stream: { - type: 'file', - qualities: {}, +export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOptions): Promise { + const sources = reorderOnIdList(ops.sourceOrder ?? [], list.sources).filter((v) => { + if (ops.media.type === 'movie') return !!v.scrapeMovie; + if (ops.media.type === 'show') return !!v.scrapeShow; + return false; + }); + const embeds = reorderOnIdList(ops.embedOrder ?? [], list.embeds); + const embedIds = embeds.map((v) => v.id); + + const contextBase: ScrapeContext = { + fetcher: ops.fetcher, + proxiedFetcher: ops.proxiedFetcher, + progress(val) { + ops.events?.update?.({ + percentage: val, + status: 'pending', + }); }, }; + + ops.events?.init?.({ + sourceIds: sources.map((v) => v.id), + }); + + for (const s of sources) { + ops.events?.start?.(s.id); + + // run source scrapers + let output: SourcererOutput | null = null; + try { + if (ops.media.type === 'movie' && s.scrapeMovie) + output = await s.scrapeMovie({ + ...contextBase, + media: ops.media, + }); + else if (ops.media.type === 'show' && s.scrapeShow) + output = await s.scrapeShow({ + ...contextBase, + media: ops.media, + }); + } catch (err) { + if (err instanceof NotFoundError) { + ops.events?.update?.({ + percentage: 100, + status: 'notfound', + }); + continue; + } + ops.events?.update?.({ + percentage: 100, + status: 'failure', + }); + // TODO log error + continue; + } + if (!output) throw new Error('Invalid media type'); + + // return stream is there are any + if (output.stream) { + return { + sourceId: s.id, + stream: output.stream, + }; + } + + if (output.embeds.length > 0) { + ops.events?.discoverEmbeds?.({ + embeds: output.embeds.map((v, i) => ({ + id: [s.id, i].join('-'), + embedScraperId: v.embedId, + })), + sourceId: s.id, + }); + } + + // run embed scrapers on listed embeds + const sortedEmbeds = output.embeds; + sortedEmbeds.sort((a, b) => embedIds.indexOf(a.embedId) - embedIds.indexOf(b.embedId)); + + for (const ind in sortedEmbeds) { + if (!Object.prototype.hasOwnProperty.call(sortedEmbeds, ind)) continue; + const e = sortedEmbeds[ind]; + const scraper = embeds.find((v) => v.id === e.embedId); + if (!scraper) throw new Error('Invalid embed returned'); + + // run embed scraper + const id = [s.id, ind].join('-'); + ops.events?.start?.(id); + let embedOutput: EmbedOutput; + try { + embedOutput = await scraper.scrape({ + ...contextBase, + url: e.url, + }); + } catch (err) { + if (err instanceof NotFoundError) { + ops.events?.update?.({ + percentage: 100, + status: 'notfound', + }); + continue; + } + ops.events?.update?.({ + percentage: 100, + status: 'failure', + }); + // TODO log error + continue; + } + + return { + sourceId: s.id, + embedId: scraper.id, + stream: embedOutput.stream, + }; + } + } + + // no providers or embeds returns streams + return null; } diff --git a/src/providers/all.ts b/src/providers/all.ts index 9ad3156..0340111 100644 --- a/src/providers/all.ts +++ b/src/providers/all.ts @@ -1,24 +1,36 @@ -import { Sourcerer } from '@/providers/base'; +import { Embed, Sourcerer } from '@/providers/base'; import { hasDuplicates, isNotNull } from '@/utils/predicates'; function gatherAllSources(): Array { + // all sources are gathered here + return []; +} + +function gatherAllEmbeds(): Array { + // all embeds are gathered here return []; } export interface ProviderList { sources: Sourcerer[]; + embeds: Embed[]; } export function getProviders(): ProviderList { const sources = gatherAllSources().filter(isNotNull); + const embeds = gatherAllEmbeds().filter(isNotNull); + const combined = [...sources, ...embeds]; - const anyDuplicateId = hasDuplicates(sources.map((v) => v.id)); - const anyDuplicateRank = hasDuplicates(sources.map((v) => v.rank)); + const anyDuplicateId = hasDuplicates(combined.map((v) => v.id)); + const anyDuplicateSourceRank = hasDuplicates(sources.map((v) => v.rank)); + const anyDuplicateEmbedRank = hasDuplicates(embeds.map((v) => v.rank)); - if (anyDuplicateId) throw new Error('Duplicate id found in sources'); - if (anyDuplicateRank) throw new Error('Duplicate rank found in sources'); + if (anyDuplicateId) throw new Error('Duplicate id found in sources/embeds'); + if (anyDuplicateSourceRank) throw new Error('Duplicate rank found in sources'); + if (anyDuplicateEmbedRank) throw new Error('Duplicate rank found in embeds'); return { sources, + embeds, }; } diff --git a/src/providers/base.ts b/src/providers/base.ts index df414f2..68c32b0 100644 --- a/src/providers/base.ts +++ b/src/providers/base.ts @@ -16,7 +16,7 @@ export type Sourcerer = { rank: number; // the higher the number, the earlier it gets put on the queue disabled?: boolean; scrapeMovie?: (input: ScrapeContext & { media: MovieMedia }) => Promise; - scrapeShow: (input: ScrapeContext & { media: ShowMedia }) => Promise; + scrapeShow?: (input: ScrapeContext & { media: ShowMedia }) => Promise; }; export function makeSourcerer(state: Sourcerer): Sourcerer | null { @@ -25,7 +25,7 @@ export function makeSourcerer(state: Sourcerer): Sourcerer | null { } export type EmbedOutput = { - stream?: Stream; + stream: Stream; }; export type Embed = { diff --git a/src/utils/list.ts b/src/utils/list.ts new file mode 100644 index 0000000..75479c4 --- /dev/null +++ b/src/utils/list.ts @@ -0,0 +1,20 @@ +export function reorderOnIdList(order: string[], list: T): T { + const copy = [...list] as T; + copy.sort((a, b) => { + const aIndex = order.indexOf(a.id); + const bIndex = order.indexOf(b.id); + + // both in order list + if (aIndex >= 0 && bIndex >= 0) return aIndex - bIndex; + + // only one in order list + // negative means order [a,b] + // positive means order [b,a] + if (aIndex < 0) return 1; // A isnt in list, so A goes later on the list + if (bIndex < 0) return -1; // B isnt in list, so B goes later on the list + + // both not in list, sort on rank + return b.rank - a.rank; + }); + return copy; +} From ffe5e4bb4f8263366d2fa562e6d15161eec31804 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sun, 27 Aug 2023 20:36:38 +0200 Subject: [PATCH 09/23] Added first source and embed --- package-lock.json | 791 ++++++++----------------- package.json | 6 + src/__test__/oof.test.ts | 5 +- src/fetchers/types.ts | 6 +- src/providers/all.ts | 6 +- src/providers/embeds/.gitkeep | 0 src/providers/embeds/upcloud.ts | 73 +++ src/providers/sources/.gitkeep | 0 src/providers/sources/flixhq.ts | 28 - src/providers/sources/flixhq/common.ts | 1 + src/providers/sources/flixhq/index.ts | 28 + src/providers/sources/flixhq/scrape.ts | 37 ++ src/providers/sources/flixhq/search.ts | 34 ++ src/utils/compare.ts | 19 + src/utils/context.ts | 4 +- tsconfig.json | 8 +- vite.config.js | 3 + 17 files changed, 451 insertions(+), 598 deletions(-) delete mode 100644 src/providers/embeds/.gitkeep create mode 100644 src/providers/embeds/upcloud.ts delete mode 100644 src/providers/sources/.gitkeep delete mode 100644 src/providers/sources/flixhq.ts create mode 100644 src/providers/sources/flixhq/common.ts create mode 100644 src/providers/sources/flixhq/index.ts create mode 100644 src/providers/sources/flixhq/scrape.ts create mode 100644 src/providers/sources/flixhq/search.ts create mode 100644 src/utils/compare.ts diff --git a/package-lock.json b/package-lock.json index b5f0095..8982fbb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,17 +1,22 @@ { - "name": "providers", - "version": "0.0.1", + "name": "@movie-web/providers", + "version": "0.0.2", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "providers", - "version": "0.0.1", + "name": "@movie-web/providers", + "version": "0.0.2", "license": "MIT", + "dependencies": { + "cheerio": "^1.0.0-rc.12", + "crypto-js": "^4.1.1", + "form-data": "^4.0.0" + }, "devDependencies": { + "@types/crypto-js": "^4.1.1", "@typescript-eslint/eslint-plugin": "^5.60.0", "@typescript-eslint/parser": "^5.60.0", - "@vitest/coverage-v8": "^0.32.2", "eslint": "^8.30.0", "eslint-config-airbnb-base": "^15.0.0", "eslint-config-prettier": "^8.5.0", @@ -20,8 +25,6 @@ "eslint-plugin-prettier": "^4.2.1", "prettier": "^2.6.2", "tsc-alias": "^1.6.7", - "tslint": "^6.1.3", - "tslint-config-prettier": "^1.18.0", "typescript": "^4.6.3", "vite": "^4.0.0", "vite-plugin-dts": "^2.3.0", @@ -29,125 +32,6 @@ "vitest": "^0.32.2" } }, - "node_modules/@ampproject/remapping": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.1.tgz", - "integrity": "sha512-lFMjJTrFL3j7L9yBxwYfCq2k6qqwHyzuUl/XBnif78PWTJYyL/dfowQHWE3sp6U6ZzqWiiIZnpTMO96zhkjwtg==", - "dev": true, - "dependencies": { - "@jridgewell/gen-mapping": "^0.3.0", - "@jridgewell/trace-mapping": "^0.3.9" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@babel/code-frame": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.22.5.tgz", - "integrity": "sha512-Xmwn266vad+6DAqEB2A6V/CcZVp62BbwVmcOJc2RPuwih1kw02TjQvWVWlcKGbBPd+8/0V5DEkOcizRGYsspYQ==", - "dev": true, - "dependencies": { - "@babel/highlight": "^7.22.5" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-identifier": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.5.tgz", - "integrity": "sha512-aJXu+6lErq8ltp+JhkJUfk1MTGyuA4v7f3pA+BJ5HLfNC6nAQ0Cpi9uOquUj8Hehg0aUiHzWQbOVJGao6ztBAQ==", - "dev": true, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/highlight": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.22.5.tgz", - "integrity": "sha512-BSKlD1hgnedS5XRnGOljZawtag7H1yPfQp0tdNJCHoH6AZ+Pcm9VvkrK59/Yy593Ypg0zMxH2BxD1VPYUQ7UIw==", - "dev": true, - "dependencies": { - "@babel/helper-validator-identifier": "^7.22.5", - "chalk": "^2.0.0", - "js-tokens": "^4.0.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/highlight/node_modules/ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "dependencies": { - "color-convert": "^1.9.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/highlight/node_modules/chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "dependencies": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/highlight/node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/@babel/highlight/node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", - "dev": true - }, - "node_modules/@babel/highlight/node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/@babel/highlight/node_modules/has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", - "dev": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/highlight/node_modules/supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dev": true, - "dependencies": { - "has-flag": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/@babel/parser": { "version": "7.22.5", "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.22.5.tgz", @@ -160,12 +44,6 @@ "node": ">=6.0.0" } }, - "node_modules/@bcoe/v8-coverage": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", - "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", - "dev": true - }, "node_modules/@esbuild/android-arm": { "version": "0.17.19", "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.17.19.tgz", @@ -607,69 +485,12 @@ "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", "dev": true }, - "node_modules/@istanbuljs/schema": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", - "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/@jridgewell/gen-mapping": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz", - "integrity": "sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==", - "dev": true, - "dependencies": { - "@jridgewell/set-array": "^1.0.1", - "@jridgewell/sourcemap-codec": "^1.4.10", - "@jridgewell/trace-mapping": "^0.3.9" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", - "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==", - "dev": true, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/set-array": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", - "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==", - "dev": true, - "engines": { - "node": ">=6.0.0" - } - }, "node_modules/@jridgewell/sourcemap-codec": { "version": "1.4.15", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", "dev": true }, - "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.18", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.18.tgz", - "integrity": "sha512-w+niJYzMHdd7USdiH2U6869nqhD2nbfZXND5Yp93qIbEmnDNk7PD48o+YchRVpzMU7M6jVCbenTR7PA1FLQ9pA==", - "dev": true, - "dependencies": { - "@jridgewell/resolve-uri": "3.1.0", - "@jridgewell/sourcemap-codec": "1.4.14" - } - }, - "node_modules/@jridgewell/trace-mapping/node_modules/@jridgewell/sourcemap-codec": { - "version": "1.4.14", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", - "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==", - "dev": true - }, "node_modules/@microsoft/api-extractor": { "version": "7.36.0", "resolved": "https://registry.npmjs.org/@microsoft/api-extractor/-/api-extractor-7.36.0.tgz", @@ -1057,6 +878,12 @@ "@types/chai": "*" } }, + "node_modules/@types/crypto-js": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@types/crypto-js/-/crypto-js-4.1.1.tgz", + "integrity": "sha512-BG7fQKZ689HIoc5h+6D2Dgq1fABRa0RbBWKBd9SP/MVRVXROflpm5fhwyATX5duFmbStzyzyycPB8qUYKDH3NA==", + "dev": true + }, "node_modules/@types/eslint": { "version": "8.40.2", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.40.2.tgz", @@ -1073,12 +900,6 @@ "integrity": "sha512-LG4opVs2ANWZ1TJoKc937iMmNstM/d0ae1vNbnBvBhqCSezgVUOzcLCqbI5elV8Vy6WKwKjaqR+zO9VKirBBCA==", "dev": true }, - "node_modules/@types/istanbul-lib-coverage": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.4.tgz", - "integrity": "sha512-z/QT1XN4K4KYuslS23k62yDIDLwLFkzxOuMplDtObz0+y7VqJCaO2o+SPwHCvLFZh7xazvvoor2tA/hPz9ee7g==", - "dev": true - }, "node_modules/@types/json-schema": { "version": "7.0.12", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.12.tgz", @@ -1457,57 +1278,6 @@ "url": "https://opencollective.com/typescript-eslint" } }, - "node_modules/@vitest/coverage-v8": { - "version": "0.32.2", - "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-0.32.2.tgz", - "integrity": "sha512-/+V3nB3fyeuuSeKxCfi6XmWjDIxpky7AWSkGVfaMjAk7di8igBwRsThLjultwIZdTDH1RAxpjmCXEfSqsMFZOA==", - "dev": true, - "dependencies": { - "@ampproject/remapping": "^2.2.1", - "@bcoe/v8-coverage": "^0.2.3", - "istanbul-lib-coverage": "^3.2.0", - "istanbul-lib-report": "^3.0.0", - "istanbul-lib-source-maps": "^4.0.1", - "istanbul-reports": "^3.1.5", - "magic-string": "^0.30.0", - "picocolors": "^1.0.0", - "std-env": "^3.3.2", - "test-exclude": "^6.0.0", - "v8-to-istanbul": "^9.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "vitest": ">=0.32.0 <1" - } - }, - "node_modules/@vitest/coverage-v8/node_modules/magic-string": { - "version": "0.30.0", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.0.tgz", - "integrity": "sha512-LA+31JYDJLs82r2ScLrlz1GjSgu66ZV518eyWT+S8VhyQn/JL0u9MeBOvQMGYiPk1DBiSN9DDMOcXvigJZaViQ==", - "dev": true, - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.4.13" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@vitest/coverage-v8/node_modules/v8-to-istanbul": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.1.0.tgz", - "integrity": "sha512-6z3GW9x8G1gd+JIIgQQQxXuiJtCXeAjp6RaPEPLv62mH3iPHPxV6W3robxtCzNErRo6ZwTmzWhsbNvjyEBKzKA==", - "dev": true, - "dependencies": { - "@jridgewell/trace-mapping": "^0.3.12", - "@types/istanbul-lib-coverage": "^2.0.1", - "convert-source-map": "^1.6.0" - }, - "engines": { - "node": ">=10.12.0" - } - }, "node_modules/@vitest/expect": { "version": "0.32.2", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-0.32.2.tgz", @@ -1853,10 +1623,7 @@ "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "dev": true, - "optional": true, - "peer": true + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "node_modules/available-typed-arrays": { "version": "1.0.5", @@ -1900,6 +1667,11 @@ "integrity": "sha512-DRQrD6gJyy8FbiE4s+bDoXS9hiW3Vbx5uCdwvcCf3zLHL+Iv7LtGHLpr+GZV8rHG8tK766FGYBwRbu8pELTt+w==", "dev": true }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" + }, "node_modules/bplist-parser": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/bplist-parser/-/bplist-parser-0.2.0.tgz", @@ -1942,15 +1714,6 @@ "optional": true, "peer": true }, - "node_modules/builtin-modules": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz", - "integrity": "sha512-wxXCdllwGhI2kCC0MnvTGYTMvnVZTvqgypkiTI8Pa5tcz2i6VqsqwYGgqwXji+4RgCzms6EajE4IxiUH6HH8nQ==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/bundle-name": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-3.0.0.tgz", @@ -2040,6 +1803,53 @@ "node": "*" } }, + "node_modules/cheerio": { + "version": "1.0.0-rc.12", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz", + "integrity": "sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "htmlparser2": "^8.0.1", + "parse5": "^7.0.0", + "parse5-htmlparser2-tree-adapter": "^7.0.0" + }, + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "dependencies": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/cheerio/node_modules/parse5": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz", + "integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==", + "dependencies": { + "entities": "^4.4.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, "node_modules/chokidar": { "version": "3.5.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", @@ -2116,9 +1926,6 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dev": true, - "optional": true, - "peer": true, "dependencies": { "delayed-stream": "~1.0.0" }, @@ -2199,12 +2006,6 @@ "integrity": "sha512-JsPKdmh8ZkmnHxDk55FZ1TqVLvEQTvoByJZRN9jzI0UjxK/QgAmsphz7PGtqgPieQZ/CQcHWXCR7ATDNhGe+YA==", "dev": true }, - "node_modules/convert-source-map": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz", - "integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==", - "dev": true - }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -2219,6 +2020,37 @@ "node": ">= 8" } }, + "node_modules/crypto-js": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.1.1.tgz", + "integrity": "sha512-o2JlM7ydqd3Qk9CA0L4NL6mTzU2sdx96a+oOfPu8Mkl/PK51vSyoi8/rQ8NknZtk44vq15lmhAj9CIAGwgeWKw==" + }, + "node_modules/css-select": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", + "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/cssom": { "version": "0.4.4", "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz", @@ -2496,22 +2328,10 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "dev": true, - "optional": true, - "peer": true, "engines": { "node": ">=0.4.0" } }, - "node_modules/diff": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", - "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", - "dev": true, - "engines": { - "node": ">=0.3.1" - } - }, "node_modules/dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -2536,6 +2356,30 @@ "node": ">=6.0.0" } }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ] + }, "node_modules/domexception": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/domexception/-/domexception-2.0.1.tgz", @@ -2561,6 +2405,33 @@ "node": ">=8" } }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", + "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, "node_modules/enhanced-resolve": { "version": "5.15.0", "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.15.0.tgz", @@ -2574,6 +2445,17 @@ "node": ">=10.13.0" } }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/es-abstract": { "version": "1.21.2", "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.21.2.tgz", @@ -3105,6 +2987,8 @@ "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", "dev": true, + "optional": true, + "peer": true, "bin": { "esparse": "bin/esparse.js", "esvalidate": "bin/esvalidate.js" @@ -3314,12 +3198,9 @@ } }, "node_modules/form-data": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", - "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", - "dev": true, - "optional": true, - "peer": true, + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -3676,11 +3557,23 @@ "node": ">=10" } }, - "node_modules/html-escaper": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", - "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", - "dev": true + "node_modules/htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } }, "node_modules/http-proxy-agent": { "version": "4.0.1", @@ -4143,56 +4036,6 @@ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "dev": true }, - "node_modules/istanbul-lib-coverage": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.0.tgz", - "integrity": "sha512-eOeJ5BHCmHYvQK7xt9GkdHuzuCGS1Y6g9Gvnx3Ym33fz/HpLRYxiS0wHNr+m/MBC8B647Xt608vCDEvhl9c6Mw==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-report": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.0.tgz", - "integrity": "sha512-wcdi+uAKzfiGT2abPpKZ0hSU1rGQjUQnLvtY5MpQ7QCTahD3VODhcu4wcfY1YtkGaDD5yuydOLINXsfbus9ROw==", - "dev": true, - "dependencies": { - "istanbul-lib-coverage": "^3.0.0", - "make-dir": "^3.0.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-source-maps": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz", - "integrity": "sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==", - "dev": true, - "dependencies": { - "debug": "^4.1.1", - "istanbul-lib-coverage": "^3.0.0", - "source-map": "^0.6.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-reports": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.5.tgz", - "integrity": "sha512-nUsEMa9pBt/NOHqbcbeJEgqIlY/K7rVWUX6Lql2orY5e9roQOthbR3vtY4zzf2orPELg80fnxxk9zUyPlgwD1w==", - "dev": true, - "dependencies": { - "html-escaper": "^2.0.0", - "istanbul-lib-report": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/jju": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/jju/-/jju-1.4.0.tgz", @@ -4208,12 +4051,6 @@ "node": ">= 0.8" } }, - "node_modules/js-tokens": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dev": true - }, "node_modules/js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", @@ -4274,6 +4111,22 @@ } } }, + "node_modules/jsdom/node_modules/form-data": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", + "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "dev": true, + "optional": true, + "peer": true, + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", @@ -4404,21 +4257,6 @@ "node": ">=12" } }, - "node_modules/make-dir": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", - "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", - "dev": true, - "dependencies": { - "semver": "^6.0.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/md5-hex": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/md5-hex/-/md5-hex-3.0.1.tgz", @@ -4463,9 +4301,6 @@ "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "dev": true, - "optional": true, - "peer": true, "engines": { "node": ">= 0.6" } @@ -4474,9 +4309,6 @@ "version": "2.1.35", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "dev": true, - "optional": true, - "peer": true, "dependencies": { "mime-db": "1.52.0" }, @@ -4514,18 +4346,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/mkdirp": { - "version": "0.5.6", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", - "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", - "dev": true, - "dependencies": { - "minimist": "^1.2.6" - }, - "bin": { - "mkdirp": "bin/cmd.js" - } - }, "node_modules/mlly": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/mlly/-/mlly-1.4.0.tgz", @@ -4608,6 +4428,17 @@ "node": ">=8" } }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, "node_modules/nwsapi": { "version": "2.2.5", "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.5.tgz", @@ -4792,6 +4623,29 @@ "optional": true, "peer": true }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.0.0.tgz", + "integrity": "sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==", + "dependencies": { + "domhandler": "^5.0.2", + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter/node_modules/parse5": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz", + "integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==", + "dependencies": { + "entities": "^4.4.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, "node_modules/path-browserify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", @@ -5490,20 +5344,6 @@ "node": ">=6" } }, - "node_modules/test-exclude": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", - "integrity": "sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==", - "dev": true, - "dependencies": { - "@istanbuljs/schema": "^0.1.2", - "glob": "^7.1.4", - "minimatch": "^3.0.4" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -5664,169 +5504,6 @@ "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", "dev": true }, - "node_modules/tslint": { - "version": "6.1.3", - "resolved": "https://registry.npmjs.org/tslint/-/tslint-6.1.3.tgz", - "integrity": "sha512-IbR4nkT96EQOvKE2PW/djGz8iGNeJ4rF2mBfiYaR/nvUWYKJhLwimoJKgjIFEIDibBtOevj7BqCRL4oHeWWUCg==", - "deprecated": "TSLint has been deprecated in favor of ESLint. Please see https://github.com/palantir/tslint/issues/4534 for more information.", - "dev": true, - "dependencies": { - "@babel/code-frame": "^7.0.0", - "builtin-modules": "^1.1.1", - "chalk": "^2.3.0", - "commander": "^2.12.1", - "diff": "^4.0.1", - "glob": "^7.1.1", - "js-yaml": "^3.13.1", - "minimatch": "^3.0.4", - "mkdirp": "^0.5.3", - "resolve": "^1.3.2", - "semver": "^5.3.0", - "tslib": "^1.13.0", - "tsutils": "^2.29.0" - }, - "bin": { - "tslint": "bin/tslint" - }, - "engines": { - "node": ">=4.8.0" - }, - "peerDependencies": { - "typescript": ">=2.3.0-dev || >=2.4.0-dev || >=2.5.0-dev || >=2.6.0-dev || >=2.7.0-dev || >=2.8.0-dev || >=2.9.0-dev || >=3.0.0-dev || >= 3.1.0-dev || >= 3.2.0-dev || >= 4.0.0-dev" - } - }, - "node_modules/tslint-config-prettier": { - "version": "1.18.0", - "resolved": "https://registry.npmjs.org/tslint-config-prettier/-/tslint-config-prettier-1.18.0.tgz", - "integrity": "sha512-xPw9PgNPLG3iKRxmK7DWr+Ea/SzrvfHtjFt5LBl61gk2UBG/DB9kCXRjv+xyIU1rUtnayLeMUVJBcMX8Z17nDg==", - "dev": true, - "bin": { - "tslint-config-prettier-check": "bin/check.js" - }, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/tslint/node_modules/ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "dependencies": { - "color-convert": "^1.9.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/tslint/node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dev": true, - "dependencies": { - "sprintf-js": "~1.0.2" - } - }, - "node_modules/tslint/node_modules/chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "dependencies": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/tslint/node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/tslint/node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", - "dev": true - }, - "node_modules/tslint/node_modules/commander": { - "version": "2.20.3", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", - "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", - "dev": true - }, - "node_modules/tslint/node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/tslint/node_modules/has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", - "dev": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/tslint/node_modules/js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", - "dev": true, - "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/tslint/node_modules/semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", - "dev": true, - "bin": { - "semver": "bin/semver" - } - }, - "node_modules/tslint/node_modules/supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dev": true, - "dependencies": { - "has-flag": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/tsutils": { - "version": "2.29.0", - "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-2.29.0.tgz", - "integrity": "sha512-g5JVHCIJwzfISaXpXE1qvNalca5Jwob6FjI4AoPlqMusJ6ftFE7IkkFoMhVLRgK+4Kx3gkzb8UZK5t5yTTvEmA==", - "dev": true, - "dependencies": { - "tslib": "^1.8.1" - }, - "peerDependencies": { - "typescript": ">=2.1.0 || >=2.1.0-dev || >=2.2.0-dev || >=2.3.0-dev || >=2.4.0-dev || >=2.5.0-dev || >=2.6.0-dev || >=2.7.0-dev || >=2.8.0-dev || >=2.9.0-dev || >= 3.0.0-dev || >= 3.1.0-dev" - } - }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", diff --git a/package.json b/package.json index 0a369ed..f139986 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "prepublishOnly": "npm test && npm run lint" }, "devDependencies": { + "@types/crypto-js": "^4.1.1", "@typescript-eslint/eslint-plugin": "^5.60.0", "@typescript-eslint/parser": "^5.60.0", "eslint": "^8.30.0", @@ -53,5 +54,10 @@ "vite-plugin-dts": "^2.3.0", "vite-plugin-eslint": "^1.8.1", "vitest": "^0.32.2" + }, + "dependencies": { + "cheerio": "^1.0.0-rc.12", + "crypto-js": "^4.1.1", + "form-data": "^4.0.0" } } diff --git a/src/__test__/oof.test.ts b/src/__test__/oof.test.ts index 6bbf556..634d1c8 100644 --- a/src/__test__/oof.test.ts +++ b/src/__test__/oof.test.ts @@ -1,8 +1,5 @@ -import { describe, expect, it } from 'vitest'; -import { LOG } from '@/testing/oof'; - describe('oof.ts', () => { it('should contain hello', () => { - expect(LOG).toContain('hello'); + expect('hello').toContain('hello'); }); }); diff --git a/src/fetchers/types.ts b/src/fetchers/types.ts index 62f78b9..ce522bc 100644 --- a/src/fetchers/types.ts +++ b/src/fetchers/types.ts @@ -1,3 +1,5 @@ +import * as FormData from 'form-data'; + export type FetcherOptions = { baseUrl?: string; headers?: Record; @@ -15,10 +17,10 @@ export type DefaultedFetcherOptions = { }; export type Fetcher = { - (url: string, ops: DefaultedFetcherOptions): T; + (url: string, ops: DefaultedFetcherOptions): Promise; }; // this feature has some quality of life features export type UseableFetcher = { - (url: string, ops?: FetcherOptions): T; + (url: string, ops?: FetcherOptions): Promise; }; diff --git a/src/providers/all.ts b/src/providers/all.ts index 0340111..b01b7e5 100644 --- a/src/providers/all.ts +++ b/src/providers/all.ts @@ -1,14 +1,16 @@ import { Embed, Sourcerer } from '@/providers/base'; +import { upcloudScraper } from '@/providers/embeds/upcloud'; +import { flixhqScraper } from '@/providers/sources/flixhq/index'; import { hasDuplicates, isNotNull } from '@/utils/predicates'; function gatherAllSources(): Array { // all sources are gathered here - return []; + return [flixhqScraper]; } function gatherAllEmbeds(): Array { // all embeds are gathered here - return []; + return [upcloudScraper]; } export interface ProviderList { diff --git a/src/providers/embeds/.gitkeep b/src/providers/embeds/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/src/providers/embeds/upcloud.ts b/src/providers/embeds/upcloud.ts new file mode 100644 index 0000000..7951b25 --- /dev/null +++ b/src/providers/embeds/upcloud.ts @@ -0,0 +1,73 @@ +import { AES, enc } from 'crypto-js'; + +import { makeEmbed } from '@/providers/base'; + +interface StreamRes { + server: number; + sources: string; + tracks: { + file: string; + kind: 'captions' | 'thumbnails'; + label: string; + }[]; +} + +function isJSON(json: string) { + try { + JSON.parse(json); + return true; + } catch { + return false; + } +} + +export const upcloudScraper = makeEmbed({ + id: 'upcloud', + name: 'UpCloud', + rank: 200, + async scrape(ctx) { + // Example url: https://dokicloud.one/embed-4/{id}?z= + const parsedUrl = new URL(ctx.url.replace('embed-5', 'embed-4')); + + const dataPath = parsedUrl.pathname.split('/'); + const dataId = dataPath[dataPath.length - 1]; + + const streamRes = await ctx.proxiedFetcher(`${parsedUrl.origin}/ajax/embed-4/getSources?id=${dataId}`, { + headers: { + Referer: parsedUrl.origin, + 'X-Requested-With': 'XMLHttpRequest', + }, + }); + + let sources: { file: string; type: string } | null = null; + + if (!isJSON(streamRes.sources)) { + const decryptionKey = JSON.parse( + await ctx.proxiedFetcher(`https://raw.githubusercontent.com/enimax-anime/key/e4/key.txt`), + ) as [number, number][]; + + let extractedKey = ''; + const sourcesArray = streamRes.sources.split(''); + for (const index of decryptionKey) { + for (let i: number = index[0]; i < index[1]; i += 1) { + extractedKey += streamRes.sources[i]; + sourcesArray[i] = ''; + } + } + + const decryptedStream = AES.decrypt(sourcesArray.join(''), extractedKey).toString(enc.Utf8); + const parsedStream = JSON.parse(decryptedStream)[0]; + if (!parsedStream) throw new Error('No stream found'); + sources = parsedStream; + } + + if (!sources) throw new Error('upcloud source not found'); + + return { + stream: { + type: 'hls', + playlist: sources.file, + }, + }; + }, +}); diff --git a/src/providers/sources/.gitkeep b/src/providers/sources/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/src/providers/sources/flixhq.ts b/src/providers/sources/flixhq.ts deleted file mode 100644 index eff0750..0000000 --- a/src/providers/sources/flixhq.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { makeSourcerer } from '@/providers/base'; - -export const flixHq = makeSourcerer({ - id: 'flixhq', - name: 'FlixHQ', - rank: 500, - - async scrapeShow(_input) { - return { - embeds: [], - stream: { - type: 'file', - qualities: { - '360': { - type: 'mp4', - url: 'blabal.mp4', - }, - }, - }, - }; - }, - - async scrapeMovie(_input) { - return { - embeds: [], - }; - }, -}); diff --git a/src/providers/sources/flixhq/common.ts b/src/providers/sources/flixhq/common.ts new file mode 100644 index 0000000..d1ce6a0 --- /dev/null +++ b/src/providers/sources/flixhq/common.ts @@ -0,0 +1 @@ +export const flixHqBase = 'https://flixhq.to'; diff --git a/src/providers/sources/flixhq/index.ts b/src/providers/sources/flixhq/index.ts new file mode 100644 index 0000000..18ceb3e --- /dev/null +++ b/src/providers/sources/flixhq/index.ts @@ -0,0 +1,28 @@ +import { makeSourcerer } from '@/providers/base'; +import { getFlixhqSourceDetails, getFlixhqSources } from '@/providers/sources/flixhq/scrape'; +import { getFlixhqId } from '@/providers/sources/flixhq/search'; +import { NotFoundError } from '@/utils/errors'; + +// TODO tv shows are available in flixHQ, just no scraper yet +export const flixhqScraper = makeSourcerer({ + id: 'flixhq', + name: 'FlixHQ', + rank: 100, + async scrapeMovie(ctx) { + const id = await getFlixhqId(ctx, ctx.media); + if (!id) throw new NotFoundError(); + + const sources = await getFlixhqSources(ctx, id); + const upcloudStream = sources.find((v) => v.embed.toLowerCase() === 'upcloud'); + if (!upcloudStream) throw new NotFoundError('upcloud stream not found for flixhq'); + + return { + embeds: [ + { + embedId: '', // TODO embed id + url: await getFlixhqSourceDetails(ctx, upcloudStream.episodeId), + }, + ], + }; + }, +}); diff --git a/src/providers/sources/flixhq/scrape.ts b/src/providers/sources/flixhq/scrape.ts new file mode 100644 index 0000000..5c25682 --- /dev/null +++ b/src/providers/sources/flixhq/scrape.ts @@ -0,0 +1,37 @@ +import { load } from 'cheerio'; + +import { flixHqBase } from '@/providers/sources/flixhq/common'; +import { ScrapeContext } from '@/utils/context'; + +export async function getFlixhqSources(ctx: ScrapeContext, id: string) { + const type = id.split('/')[0]; + const episodeParts = id.split('-'); + const episodeId = episodeParts[episodeParts.length - 1]; + + const data = await ctx.proxiedFetcher(`/ajax/${type}/episodes/${episodeId}`, { + baseUrl: flixHqBase, + }); + const doc = load(data); + const sourceLinks = doc('.nav-item > a') + .toArray() + .map((el) => { + const query = doc(el); + const embedTitle = query.attr('title'); + const linkId = query.attr('data-linkid'); + if (!embedTitle || !linkId) throw new Error('invalid sources'); + return { + embed: embedTitle, + episodeId: linkId, + }; + }); + + return sourceLinks; +} + +export async function getFlixhqSourceDetails(ctx: ScrapeContext, sourceId: string): Promise { + const jsonData = await ctx.proxiedFetcher>(`/ajax/sources/${sourceId}`, { + baseUrl: flixHqBase, + }); + + return jsonData.link; +} diff --git a/src/providers/sources/flixhq/search.ts b/src/providers/sources/flixhq/search.ts new file mode 100644 index 0000000..569d2d1 --- /dev/null +++ b/src/providers/sources/flixhq/search.ts @@ -0,0 +1,34 @@ +import { load } from 'cheerio'; + +import { MovieMedia } from '@/main/media'; +import { flixHqBase } from '@/providers/sources/flixhq/common'; +import { compareMedia } from '@/utils/compare'; +import { ScrapeContext } from '@/utils/context'; + +export async function getFlixhqId(ctx: ScrapeContext, media: MovieMedia): Promise { + const searchResults = await ctx.proxiedFetcher(`/search/${media.title.replaceAll(/[^a-z0-9A-Z]/g, '-')}`, { + baseUrl: flixHqBase, + }); + + const doc = load(searchResults); + const items = doc('.film_list-wrap > div.flw-item') + .toArray() + .map((el) => { + const query = doc(el); + const id = query.find('div.film-poster > a').attr('href')?.slice(1); + const title = query.find('div.film-detail > h2 > a').attr('title'); + const year = query.find('div.film-detail > div.fd-infor > span:nth-child(1)').text(); + + if (!id || !title || !year) return null; + return { + id, + title, + year: +year, + }; + }); + + const matchingItem = items.find((v) => v && compareMedia(media, v.title, v.year)); + + if (!matchingItem) return null; + return matchingItem.id; +} diff --git a/src/utils/compare.ts b/src/utils/compare.ts new file mode 100644 index 0000000..8cce7da --- /dev/null +++ b/src/utils/compare.ts @@ -0,0 +1,19 @@ +import { CommonMedia } from '@/main/media'; + +export function normalizeTitle(title: string): string { + return title + .trim() + .toLowerCase() + .replace(/['":]/g, '') + .replace(/[^a-zA-Z0-9]+/g, '_'); +} + +export function compareTitle(a: string, b: string): boolean { + return normalizeTitle(a) === normalizeTitle(b); +} + +export function compareMedia(media: CommonMedia, title: string, releaseYear?: number): boolean { + // if no year is provided, count as if its the correct year + const isSameYear = releaseYear === undefined ? true : media.releaseYear === releaseYear; + return compareTitle(media.title, title) && isSameYear; +} diff --git a/src/utils/context.ts b/src/utils/context.ts index 5d89f96..5a9664a 100644 --- a/src/utils/context.ts +++ b/src/utils/context.ts @@ -1,8 +1,8 @@ import { UseableFetcher } from '@/fetchers/types'; export type ScrapeContext = { - proxiedFetcher: UseableFetcher; - fetcher: UseableFetcher; + proxiedFetcher: (...params: Parameters>) => ReturnType>; + fetcher: (...params: Parameters>) => ReturnType>; progress(val: number): void; }; diff --git a/tsconfig.json b/tsconfig.json index e1c2373..d8fea61 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { - "target": "es2018", - "lib": ["es2018", "DOM"], + "target": "es2021", + "lib": ["es2021"], "module": "commonjs", "declaration": true, "outDir": "./lib", @@ -9,8 +9,10 @@ "baseUrl": "src", "experimentalDecorators": true, "isolatedModules": false, + "types": ["vitest/globals"], "paths": { - "@/*": ["./*"] + "@/*": ["./*"], + "@entrypoint": ["./index.ts"] } }, "include": ["src"], diff --git a/vite.config.js b/vite.config.js index 7fa6d57..5217c56 100644 --- a/vite.config.js +++ b/vite.config.js @@ -27,4 +27,7 @@ module.exports = defineConfig({ fileName: 'providers', }, }, + test: { + globals: true, + }, }); From be9e8d6da206b710a842576f1da8a5aeb6ff88bc Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sun, 27 Aug 2023 20:47:48 +0200 Subject: [PATCH 10/23] Fix build --- package-lock.json | 87 ++++++++++++++++++++++++++++++++++- package.json | 3 +- src/__test__/oof.test.ts | 2 + src/fetchers/standardFetch.ts | 2 + tsconfig.json | 2 +- vite.config.js | 14 ++---- 6 files changed, 96 insertions(+), 14 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8982fbb..014124f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,8 @@ "dependencies": { "cheerio": "^1.0.0-rc.12", "crypto-js": "^4.1.1", - "form-data": "^4.0.0" + "form-data": "^4.0.0", + "node-fetch": "^3.3.2" }, "devDependencies": { "@types/crypto-js": "^4.1.1", @@ -2081,6 +2082,14 @@ "optional": true, "peer": true }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "engines": { + "node": ">= 12" + } + }, "node_modules/data-urls": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz", @@ -3129,6 +3138,28 @@ "reusify": "^1.0.4" } }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, "node_modules/file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -3210,6 +3241,17 @@ "node": ">= 6" } }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/fs-extra": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", @@ -4407,6 +4449,41 @@ "integrity": "sha512-Tj+HTDSJJKaZnfiuw+iaF9skdPpTo2GtEly5JHnWV/hfv2Qj/9RKsGISQtLh2ox3l5EAGw487hnBee0sIJ6v2g==", "dev": true }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -5907,6 +5984,14 @@ "node": ">=10" } }, + "node_modules/web-streams-polyfill": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", + "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==", + "engines": { + "node": ">= 8" + } + }, "node_modules/webidl-conversions": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-6.1.0.tgz", diff --git a/package.json b/package.json index f139986..4d6acc8 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "dependencies": { "cheerio": "^1.0.0-rc.12", "crypto-js": "^4.1.1", - "form-data": "^4.0.0" + "form-data": "^4.0.0", + "node-fetch": "^3.3.2" } } diff --git a/src/__test__/oof.test.ts b/src/__test__/oof.test.ts index 634d1c8..f17bde3 100644 --- a/src/__test__/oof.test.ts +++ b/src/__test__/oof.test.ts @@ -1,3 +1,5 @@ +import { describe, it, expect } from "vitest"; + describe('oof.ts', () => { it('should contain hello', () => { expect('hello').toContain('hello'); diff --git a/src/fetchers/standardFetch.ts b/src/fetchers/standardFetch.ts index 54ad91e..a4000eb 100644 --- a/src/fetchers/standardFetch.ts +++ b/src/fetchers/standardFetch.ts @@ -1,3 +1,5 @@ +import fetch from 'node-fetch'; + import { makeFullUrl } from '@/fetchers/common'; import { Fetcher } from '@/fetchers/types'; diff --git a/tsconfig.json b/tsconfig.json index d8fea61..d3fea2f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -9,7 +9,7 @@ "baseUrl": "src", "experimentalDecorators": true, "isolatedModules": false, - "types": ["vitest/globals"], + "skipLibCheck": true, "paths": { "@/*": ["./*"], "@entrypoint": ["./index.ts"] diff --git a/vite.config.js b/vite.config.js index 5217c56..d3f922b 100644 --- a/vite.config.js +++ b/vite.config.js @@ -6,12 +6,7 @@ const dts = require('vite-plugin-dts'); const main = path.resolve(__dirname, 'src/index.ts'); module.exports = defineConfig({ - plugins: [ - eslint(), - dts({ - include: [main], - }), - ], + plugins: [eslint(), dts({})], resolve: { alias: { '@': path.resolve(__dirname, './src'), @@ -23,11 +18,8 @@ module.exports = defineConfig({ lib: { entry: main, - name: 'providers', - fileName: 'providers', + name: 'index', + fileName: 'index', }, }, - test: { - globals: true, - }, }); From 391432c1ba51f2ad93dd5e83e8a104e241d0e363 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sun, 27 Aug 2023 20:57:24 +0200 Subject: [PATCH 11/23] fix flixhq scraper and organize todos --- README.md | 27 +++++++++++++++------------ src/providers/all.ts | 10 +++++----- src/providers/base.ts | 6 ++---- src/providers/sources/flixhq/index.ts | 3 ++- src/utils/predicates.ts | 4 ---- 5 files changed, 24 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index b5fa78a..75c4979 100644 --- a/README.md +++ b/README.md @@ -9,16 +9,19 @@ features: > This package is still WIP -> TODO documentation: examples for nodejs + browser +Todos: + - add tests (integration, unit tests) + - running individual scrapers + - finish fetchers: + - make baseUrl param work + - proper serialization (with content-type headers) for standard fetcher + - automatically parse json + - error logging for failed scrapers + - make the lib not compile into one file, keep dependency structure -> TODO documentation: how to use + usecases - -> TODO documentation: examples on how to make a custom fetcher - -> TODO functionality: running individual scrapers - -> TODO functionality: choose environment (for browser, for native) - -> TODO content: add all scrapers/providers - -> TODO tests: add tests +Future todos: + - docs: examples for nodejs + browser + - docs: how to use + usecases + - docs: examples for custom fetcher + - choose an output environment (for browser or for native) + - flixhq show support diff --git a/src/providers/all.ts b/src/providers/all.ts index b01b7e5..c2c924d 100644 --- a/src/providers/all.ts +++ b/src/providers/all.ts @@ -1,14 +1,14 @@ import { Embed, Sourcerer } from '@/providers/base'; import { upcloudScraper } from '@/providers/embeds/upcloud'; import { flixhqScraper } from '@/providers/sources/flixhq/index'; -import { hasDuplicates, isNotNull } from '@/utils/predicates'; +import { hasDuplicates } from '@/utils/predicates'; -function gatherAllSources(): Array { +function gatherAllSources(): Array { // all sources are gathered here return [flixhqScraper]; } -function gatherAllEmbeds(): Array { +function gatherAllEmbeds(): Array { // all embeds are gathered here return [upcloudScraper]; } @@ -19,8 +19,8 @@ export interface ProviderList { } export function getProviders(): ProviderList { - const sources = gatherAllSources().filter(isNotNull); - const embeds = gatherAllEmbeds().filter(isNotNull); + const sources = gatherAllSources().filter((v) => !v?.disabled); + const embeds = gatherAllEmbeds().filter((v) => !v?.disabled); const combined = [...sources, ...embeds]; const anyDuplicateId = hasDuplicates(combined.map((v) => v.id)); diff --git a/src/providers/base.ts b/src/providers/base.ts index 68c32b0..4942ffd 100644 --- a/src/providers/base.ts +++ b/src/providers/base.ts @@ -19,8 +19,7 @@ export type Sourcerer = { scrapeShow?: (input: ScrapeContext & { media: ShowMedia }) => Promise; }; -export function makeSourcerer(state: Sourcerer): Sourcerer | null { - if (state.disabled) return null; +export function makeSourcerer(state: Sourcerer): Sourcerer { return state; } @@ -36,7 +35,6 @@ export type Embed = { scrape: (input: EmbedScrapeContext) => Promise; }; -export function makeEmbed(state: Embed): Embed | null { - if (state.disabled) return null; +export function makeEmbed(state: Embed): Embed { return state; } diff --git a/src/providers/sources/flixhq/index.ts b/src/providers/sources/flixhq/index.ts index 18ceb3e..f847502 100644 --- a/src/providers/sources/flixhq/index.ts +++ b/src/providers/sources/flixhq/index.ts @@ -1,4 +1,5 @@ import { makeSourcerer } from '@/providers/base'; +import { upcloudScraper } from '@/providers/embeds/upcloud'; import { getFlixhqSourceDetails, getFlixhqSources } from '@/providers/sources/flixhq/scrape'; import { getFlixhqId } from '@/providers/sources/flixhq/search'; import { NotFoundError } from '@/utils/errors'; @@ -19,7 +20,7 @@ export const flixhqScraper = makeSourcerer({ return { embeds: [ { - embedId: '', // TODO embed id + embedId: upcloudScraper.id, url: await getFlixhqSourceDetails(ctx, upcloudStream.episodeId), }, ], diff --git a/src/utils/predicates.ts b/src/utils/predicates.ts index 03e7469..f581b2f 100644 --- a/src/utils/predicates.ts +++ b/src/utils/predicates.ts @@ -1,7 +1,3 @@ -export function isNotNull(value: T | null | undefined): value is T { - return value !== null && value !== undefined; -} - export function hasDuplicates(values: Array): boolean { return new Set(values).size !== values.length; } From bcf312d1b3e06bc34ab2d3447771dceedceb84a2 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 5 Sep 2023 20:57:10 +0200 Subject: [PATCH 12/23] unit tests for provider checks, provider listings, utils and provider meta --- src/__test__/oof.test.ts | 7 -- src/__test__/providerTests.ts | 122 ++++++++++++++++++++++++++ src/__test__/providers/checks.test.ts | 63 +++++++++++++ src/__test__/runner/list.test.ts | 121 +++++++++++++++++++++++++ src/__test__/runner/meta.test.ts | 50 +++++++++++ src/__test__/utils/list.test.ts | 54 ++++++++++++ src/main/builder.ts | 2 +- src/main/meta.ts | 58 +++++++----- src/providers/all.ts | 29 +----- src/providers/get.ts | 27 ++++++ src/utils/list.ts | 4 +- 11 files changed, 476 insertions(+), 61 deletions(-) delete mode 100644 src/__test__/oof.test.ts create mode 100644 src/__test__/providerTests.ts create mode 100644 src/__test__/providers/checks.test.ts create mode 100644 src/__test__/runner/list.test.ts create mode 100644 src/__test__/runner/meta.test.ts create mode 100644 src/__test__/utils/list.test.ts create mode 100644 src/providers/get.ts diff --git a/src/__test__/oof.test.ts b/src/__test__/oof.test.ts deleted file mode 100644 index f17bde3..0000000 --- a/src/__test__/oof.test.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { describe, it, expect } from "vitest"; - -describe('oof.ts', () => { - it('should contain hello', () => { - expect('hello').toContain('hello'); - }); -}); diff --git a/src/__test__/providerTests.ts b/src/__test__/providerTests.ts new file mode 100644 index 0000000..a8c0fa8 --- /dev/null +++ b/src/__test__/providerTests.ts @@ -0,0 +1,122 @@ +// eslint-disable-next-line import/no-extraneous-dependencies +import { vi } from 'vitest'; + +import { gatherAllEmbeds, gatherAllSources } from '@/providers/all'; +import { Embed, Sourcerer } from '@/providers/base'; + +export function makeProviderMocks() { + const embedsMock = vi.fn, ReturnType>(); + const sourcesMock = vi.fn, ReturnType>(); + return { + gatherAllEmbeds: embedsMock, + gatherAllSources: sourcesMock, + }; +} + +const sourceA = { + id: 'a', + rank: 1, + disabled: false, +} as Sourcerer; +const sourceB = { + id: 'b', + rank: 2, + disabled: false, +} as Sourcerer; +const sourceCDisabled = { + id: 'c', + rank: 3, + disabled: true, +} as Sourcerer; +const sourceAHigherRank = { + id: 'a', + rank: 100, + disabled: false, +} as Sourcerer; +const sourceGSameRankAsA = { + id: 'g', + rank: 1, + disabled: false, +} as Sourcerer; +const fullSourceYMovie = { + id: 'y', + name: 'Y', + rank: 105, + scrapeMovie: vi.fn(), +} as Sourcerer; +const fullSourceYShow = { + id: 'y', + name: 'Y', + rank: 105, + scrapeShow: vi.fn(), +} as Sourcerer; +const fullSourceZBoth = { + id: 'z', + name: 'Z', + rank: 106, + scrapeMovie: vi.fn(), + scrapeShow: vi.fn(), +} as Sourcerer; + +const embedD = { + id: 'd', + rank: 4, + disabled: false, +} as Embed; +const embedA = { + id: 'a', + rank: 5, + disabled: false, +} as Embed; +const embedEDisabled = { + id: 'e', + rank: 6, + disabled: true, +} as Embed; +const embedDHigherRank = { + id: 'd', + rank: 4000, + disabled: false, +} as Embed; +const embedFSameRankAsA = { + id: 'f', + rank: 5, + disabled: false, +} as Embed; +const embedHSameRankAsSourceA = { + id: 'h', + rank: 1, + disabled: false, +} as Embed; +const fullEmbedX = { + id: 'x', + name: 'X', + rank: 104, +} as Embed; +const fullEmbedZ = { + id: 'z', + name: 'Z', + rank: 109, +} as Embed; + +export const mockSources = { + sourceA, + sourceB, + sourceCDisabled, + sourceAHigherRank, + sourceGSameRankAsA, + fullSourceYMovie, + fullSourceYShow, + fullSourceZBoth, +}; + +export const mockEmbeds = { + embedA, + embedD, + embedDHigherRank, + embedEDisabled, + embedFSameRankAsA, + embedHSameRankAsSourceA, + fullEmbedX, + fullEmbedZ, +}; diff --git a/src/__test__/providers/checks.test.ts b/src/__test__/providers/checks.test.ts new file mode 100644 index 0000000..71bef7c --- /dev/null +++ b/src/__test__/providers/checks.test.ts @@ -0,0 +1,63 @@ +import { mockEmbeds, mockSources } from '@/__test__/providerTests'; +import { getProviders } from '@/providers/get'; +import { vi, describe, it, expect, afterEach } from 'vitest'; + +const mocks = await vi.hoisted(async () => (await import('@/__test__/providerTests')).makeProviderMocks()); +vi.mock('@/providers/all', () => mocks); + +describe('getProviders()', () => { + afterEach(() => { + vi.clearAllMocks(); + }); + + it('should return providers', () => { + mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD]); + mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceB]); + expect(getProviders()).toEqual({ + sources: [mockSources.sourceA, mockSources.sourceB], + embeds: [mockEmbeds.embedD], + }); + }); + + it('should filter out disabled providers', () => { + mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedEDisabled]); + mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceCDisabled, mockSources.sourceB]); + expect(getProviders()).toEqual({ + sources: [mockSources.sourceA, mockSources.sourceB], + embeds: [mockEmbeds.embedD], + }); + }); + + it('should throw on duplicate ids in sources', () => { + mocks.gatherAllEmbeds.mockReturnValue([]); + mocks.gatherAllSources.mockReturnValue([mockSources.sourceAHigherRank, mockSources.sourceA, mockSources.sourceB]); + expect(() => getProviders()).toThrowError(); + }); + + it('should throw on duplicate ids in embeds', () => { + mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedDHigherRank, mockEmbeds.embedA]); + mocks.gatherAllSources.mockReturnValue([]); + expect(() => getProviders()).toThrowError(); + }); + + it('should throw on duplicate ids between sources and embeds', () => { + mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedA]); + mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceB]); + expect(() => getProviders()).toThrowError(); + }); + + it('should throw on duplicate rank between sources and embeds', () => { + mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedA]); + mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceB]); + expect(() => getProviders()).toThrowError(); + }); + + it('should not throw with same rank between sources and embeds', () => { + mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedHSameRankAsSourceA]); + mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceB]); + expect(getProviders()).toEqual({ + sources: [mockSources.sourceA, mockSources.sourceB], + embeds: [mockEmbeds.embedD, mockEmbeds.embedHSameRankAsSourceA], + }); + }); +}); diff --git a/src/__test__/runner/list.test.ts b/src/__test__/runner/list.test.ts new file mode 100644 index 0000000..7c46648 --- /dev/null +++ b/src/__test__/runner/list.test.ts @@ -0,0 +1,121 @@ +import { mockEmbeds, mockSources } from '@/__test__/providerTests'; +import { makeProviders } from '@/main/builder'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +const mocks = await vi.hoisted(async () => (await import('@/__test__/providerTests')).makeProviderMocks()); +vi.mock('@/providers/all', () => mocks); + +describe('ProviderControls.listSources()', () => { + afterEach(() => { + vi.clearAllMocks(); + }); + + it('should return the source with movie type', () => { + mocks.gatherAllSources.mockReturnValue([mockSources.fullSourceYMovie]); + mocks.gatherAllEmbeds.mockReturnValue([]); + const p = makeProviders({ + fetcher: null as any, + }); + expect(p.listSources()).toEqual([ + { + type: 'source', + id: 'y', + rank: mockSources.fullSourceYMovie.rank, + name: 'Y', + mediaTypes: ['movie'], + }, + ]); + }); + + it('should return the source with show type', () => { + mocks.gatherAllSources.mockReturnValue([mockSources.fullSourceYShow]); + mocks.gatherAllEmbeds.mockReturnValue([]); + const p = makeProviders({ + fetcher: null as any, + }); + expect(p.listSources()).toEqual([ + { + type: 'source', + id: 'y', + rank: mockSources.fullSourceYShow.rank, + name: 'Y', + mediaTypes: ['show'], + }, + ]); + }); + + it('should return the source with both types', () => { + mocks.gatherAllSources.mockReturnValue([mockSources.fullSourceZBoth]); + mocks.gatherAllEmbeds.mockReturnValue([]); + const p = makeProviders({ + fetcher: null as any, + }); + expect(p.listSources()).toEqual([ + { + type: 'source', + id: 'z', + rank: mockSources.fullSourceZBoth.rank, + name: 'Z', + mediaTypes: ['movie', 'show'], + }, + ]); + }); + + it('should return the sources in correct order', () => { + mocks.gatherAllSources.mockReturnValue([mockSources.fullSourceYMovie, mockSources.fullSourceZBoth]); + mocks.gatherAllEmbeds.mockReturnValue([]); + const p1 = makeProviders({ + fetcher: null as any, + }); + const l1 = p1.listSources(); + expect(l1.map((v) => v.id).join(',')).toEqual('z,y'); + + mocks.gatherAllSources.mockReturnValue([mockSources.fullSourceZBoth, mockSources.fullSourceYMovie]); + mocks.gatherAllEmbeds.mockReturnValue([]); + const p2 = makeProviders({ + fetcher: null as any, + }); + const l2 = p2.listSources(); + expect(l2.map((v) => v.id).join(',')).toEqual('z,y'); + }); +}); + +describe('ProviderControls.getAllEmbedMetaSorted()', () => { + afterEach(() => { + vi.clearAllMocks(); + }); + + it('should return the correct embed format', () => { + mocks.gatherAllSources.mockReturnValue([]); + mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.fullEmbedX]); + const p = makeProviders({ + fetcher: null as any, + }); + expect(p.listEmbeds()).toEqual([ + { + type: 'embed', + id: 'x', + rank: mockEmbeds.fullEmbedX.rank, + name: 'X', + }, + ]); + }); + + it('should return the embeds in correct order', () => { + mocks.gatherAllSources.mockReturnValue([]); + mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.fullEmbedX, mockEmbeds.fullEmbedZ]); + const p1 = makeProviders({ + fetcher: null as any, + }); + const l1 = p1.listEmbeds(); + expect(l1.map((v) => v.id).join(',')).toEqual('z,x'); + + mocks.gatherAllSources.mockReturnValue([]); + mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.fullEmbedZ, mockEmbeds.fullEmbedX]); + const p2 = makeProviders({ + fetcher: null as any, + }); + const l2 = p2.listEmbeds(); + expect(l2.map((v) => v.id).join(',')).toEqual('z,x'); + }); +}); diff --git a/src/__test__/runner/meta.test.ts b/src/__test__/runner/meta.test.ts new file mode 100644 index 0000000..4db1fdc --- /dev/null +++ b/src/__test__/runner/meta.test.ts @@ -0,0 +1,50 @@ +import { mockEmbeds, mockSources } from '@/__test__/providerTests'; +import { makeProviders } from '@/main/builder'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +const mocks = await vi.hoisted(async () => (await import('@/__test__/providerTests')).makeProviderMocks()); +vi.mock('@/providers/all', () => mocks); + +describe('ProviderControls.getMetadata()', () => { + afterEach(() => { + vi.clearAllMocks(); + }); + + it('should return null if not found', () => { + mocks.gatherAllSources.mockReturnValue([]); + mocks.gatherAllEmbeds.mockReturnValue([]); + const p = makeProviders({ + fetcher: null as any, + }); + expect(p.getMetadata(':)')).toEqual(null); + }); + + it('should return correct source meta', () => { + mocks.gatherAllSources.mockReturnValue([mockSources.fullSourceZBoth]); + mocks.gatherAllEmbeds.mockReturnValue([]); + const p = makeProviders({ + fetcher: null as any, + }); + expect(p.getMetadata(mockSources.fullSourceZBoth.id)).toEqual({ + type: 'source', + id: 'z', + name: 'Z', + rank: mockSources.fullSourceZBoth.rank, + mediaTypes: ['movie', 'show'], + }); + }); + + it('should return correct embed meta', () => { + mocks.gatherAllSources.mockReturnValue([]); + mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.fullEmbedX]); + const p = makeProviders({ + fetcher: null as any, + }); + expect(p.getMetadata(mockEmbeds.fullEmbedX.id)).toEqual({ + type: 'embed', + id: 'x', + name: 'X', + rank: mockEmbeds.fullEmbedX.rank, + }); + }); +}); diff --git a/src/__test__/utils/list.test.ts b/src/__test__/utils/list.test.ts new file mode 100644 index 0000000..9ebf93e --- /dev/null +++ b/src/__test__/utils/list.test.ts @@ -0,0 +1,54 @@ +import { reorderOnIdList } from "@/utils/list"; +import { describe, it, expect } from "vitest"; + +function list(def: string) { + return def.split(",").map(v=>({ + rank: parseInt(v), + id: v, + })) +} + +function expectListToEqual(l1: ReturnType, l2: ReturnType) { + function flatten(l: ReturnType) { + return l.map(v=>v.id).join(","); + } + expect(flatten(l1)).toEqual(flatten(l2)); +} + +describe('reorderOnIdList()', () => { + it('should reorder based on rank', () => { + const l = list('2,1,4,3'); + const sortedList = list('4,3,2,1') + expectListToEqual(reorderOnIdList([], l), sortedList); + }); + + it('should work with empty input', () => { + expectListToEqual(reorderOnIdList([], []), []); + }); + + it('should reorder based on id list', () => { + const l = list('4,2,1,3'); + const sortedList = list('4,3,2,1') + expectListToEqual(reorderOnIdList(["4","3","2","1"], l), sortedList); + }); + + it('should reorder based on id list and rank second', () => { + const l = list('4,2,1,3'); + const sortedList = list('4,3,2,1') + expectListToEqual(reorderOnIdList(["4","3"], l), sortedList); + }); + + it('should work with only one item', () => { + const l = list('1'); + const sortedList = list('1') + expectListToEqual(reorderOnIdList(["1"], l), sortedList); + expectListToEqual(reorderOnIdList([], l), sortedList); + }); + + it('should not affect original list', () => { + const l = list('4,3,2,1'); + const unsortedList = list('4,3,2,1') + reorderOnIdList([], l); + expectListToEqual(l, unsortedList); + }); +}); diff --git a/src/main/builder.ts b/src/main/builder.ts index 2cf50fe..7035e6d 100644 --- a/src/main/builder.ts +++ b/src/main/builder.ts @@ -4,7 +4,7 @@ import { FullScraperEvents } from '@/main/events'; import { ScrapeMedia } from '@/main/media'; import { MetaOutput, getAllEmbedMetaSorted, getAllSourceMetaSorted, getSpecificId } from '@/main/meta'; import { RunOutput, runAllProviders } from '@/main/runner'; -import { getProviders } from '@/providers/all'; +import { getProviders } from '@/providers/get'; export interface ProviderBuilderOptions { // fetcher, every web request gets called through here diff --git a/src/main/meta.ts b/src/main/meta.ts index cbd8297..5696183 100644 --- a/src/main/meta.ts +++ b/src/main/meta.ts @@ -1,5 +1,6 @@ import { MediaTypes } from '@/main/media'; -import { ProviderList } from '@/providers/all'; +import { Embed, Sourcerer } from '@/providers/base'; +import { ProviderList } from '@/providers/get'; export type MetaOutput = { type: 'embed' | 'source'; @@ -9,36 +10,45 @@ export type MetaOutput = { mediaTypes?: Array; }; -export function getAllSourceMetaSorted(list: ProviderList): MetaOutput[] { - return list.sources - .sort((a, b) => b.rank - a.rank) - .map((v) => { - const types: Array = []; - if (v.scrapeMovie) types.push('movie'); - if (v.scrapeShow) types.push('show'); - return { - type: 'source', - id: v.id, - rank: v.rank, - name: v.name, - mediaTypes: types, - }; - }); +function formatSourceMeta(v: Sourcerer): MetaOutput { + const types: Array = []; + if (v.scrapeMovie) types.push('movie'); + if (v.scrapeShow) types.push('show'); + return { + type: 'source', + id: v.id, + rank: v.rank, + name: v.name, + mediaTypes: types, + }; } -export function getAllEmbedMetaSorted(_list: ProviderList): MetaOutput[] { - return []; +function formatEmbedMeta(v: Embed): MetaOutput { + return { + type: 'embed', + id: v.id, + rank: v.rank, + name: v.name, + }; +} + +export function getAllSourceMetaSorted(list: ProviderList): MetaOutput[] { + return list.sources.sort((a, b) => b.rank - a.rank).map(formatSourceMeta); +} + +export function getAllEmbedMetaSorted(list: ProviderList): MetaOutput[] { + return list.embeds.sort((a, b) => b.rank - a.rank).map(formatEmbedMeta); } export function getSpecificId(list: ProviderList, id: string): MetaOutput | null { const foundSource = list.sources.find((v) => v.id === id); if (foundSource) { - return { - type: 'source', - id: foundSource.id, - name: foundSource.name, - rank: foundSource.rank, - }; + return formatSourceMeta(foundSource); + } + + const foundEmbed = list.embeds.find((v) => v.id === id); + if (foundEmbed) { + return formatEmbedMeta(foundEmbed); } return null; diff --git a/src/providers/all.ts b/src/providers/all.ts index c2c924d..e9f7c1d 100644 --- a/src/providers/all.ts +++ b/src/providers/all.ts @@ -1,38 +1,13 @@ import { Embed, Sourcerer } from '@/providers/base'; import { upcloudScraper } from '@/providers/embeds/upcloud'; import { flixhqScraper } from '@/providers/sources/flixhq/index'; -import { hasDuplicates } from '@/utils/predicates'; -function gatherAllSources(): Array { +export function gatherAllSources(): Array { // all sources are gathered here return [flixhqScraper]; } -function gatherAllEmbeds(): Array { +export function gatherAllEmbeds(): Array { // all embeds are gathered here return [upcloudScraper]; } - -export interface ProviderList { - sources: Sourcerer[]; - embeds: Embed[]; -} - -export function getProviders(): ProviderList { - const sources = gatherAllSources().filter((v) => !v?.disabled); - const embeds = gatherAllEmbeds().filter((v) => !v?.disabled); - const combined = [...sources, ...embeds]; - - const anyDuplicateId = hasDuplicates(combined.map((v) => v.id)); - const anyDuplicateSourceRank = hasDuplicates(sources.map((v) => v.rank)); - const anyDuplicateEmbedRank = hasDuplicates(embeds.map((v) => v.rank)); - - if (anyDuplicateId) throw new Error('Duplicate id found in sources/embeds'); - if (anyDuplicateSourceRank) throw new Error('Duplicate rank found in sources'); - if (anyDuplicateEmbedRank) throw new Error('Duplicate rank found in embeds'); - - return { - sources, - embeds, - }; -} diff --git a/src/providers/get.ts b/src/providers/get.ts new file mode 100644 index 0000000..689617f --- /dev/null +++ b/src/providers/get.ts @@ -0,0 +1,27 @@ +import { gatherAllEmbeds, gatherAllSources } from '@/providers/all'; +import { Embed, Sourcerer } from '@/providers/base'; +import { hasDuplicates } from '@/utils/predicates'; + +export interface ProviderList { + sources: Sourcerer[]; + embeds: Embed[]; +} + +export function getProviders(): ProviderList { + const sources = gatherAllSources().filter((v) => !v?.disabled); + const embeds = gatherAllEmbeds().filter((v) => !v?.disabled); + const combined = [...sources, ...embeds]; + + const anyDuplicateId = hasDuplicates(combined.map((v) => v.id)); + const anyDuplicateSourceRank = hasDuplicates(sources.map((v) => v.rank)); + const anyDuplicateEmbedRank = hasDuplicates(embeds.map((v) => v.rank)); + + if (anyDuplicateId) throw new Error('Duplicate id found in sources/embeds'); + if (anyDuplicateSourceRank) throw new Error('Duplicate rank found in sources'); + if (anyDuplicateEmbedRank) throw new Error('Duplicate rank found in embeds'); + + return { + sources, + embeds, + }; +} diff --git a/src/utils/list.ts b/src/utils/list.ts index 75479c4..8d619fb 100644 --- a/src/utils/list.ts +++ b/src/utils/list.ts @@ -10,8 +10,8 @@ export function reorderOnIdList(order: // only one in order list // negative means order [a,b] // positive means order [b,a] - if (aIndex < 0) return 1; // A isnt in list, so A goes later on the list - if (bIndex < 0) return -1; // B isnt in list, so B goes later on the list + if (bIndex >= 0) return 1; // A isnt in list but B is, so A goes later on the list + if (aIndex >= 0) return -1; // B isnt in list but A is, so B goes later on the list // both not in list, sort on rank return b.rank - a.rank; From 7d7c6865e4d894b8c838e0dfc9b81fdf55ace63b Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 5 Sep 2023 21:29:59 +0200 Subject: [PATCH 13/23] fix ts config for tests, so my IDE doesnt complain --- src/__test__/providers/checks.test.ts | 2 +- src/__test__/runner/list.test.ts | 2 +- src/__test__/runner/meta.test.ts | 2 +- src/__test__/tsconfig.json | 21 +++++++++++++++++++++ 4 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 src/__test__/tsconfig.json diff --git a/src/__test__/providers/checks.test.ts b/src/__test__/providers/checks.test.ts index 71bef7c..bc89a20 100644 --- a/src/__test__/providers/checks.test.ts +++ b/src/__test__/providers/checks.test.ts @@ -2,7 +2,7 @@ import { mockEmbeds, mockSources } from '@/__test__/providerTests'; import { getProviders } from '@/providers/get'; import { vi, describe, it, expect, afterEach } from 'vitest'; -const mocks = await vi.hoisted(async () => (await import('@/__test__/providerTests')).makeProviderMocks()); +const mocks = await vi.hoisted(async () => (await import('../providerTests.ts')).makeProviderMocks()); vi.mock('@/providers/all', () => mocks); describe('getProviders()', () => { diff --git a/src/__test__/runner/list.test.ts b/src/__test__/runner/list.test.ts index 7c46648..b10eddb 100644 --- a/src/__test__/runner/list.test.ts +++ b/src/__test__/runner/list.test.ts @@ -2,7 +2,7 @@ import { mockEmbeds, mockSources } from '@/__test__/providerTests'; import { makeProviders } from '@/main/builder'; import { afterEach, describe, expect, it, vi } from 'vitest'; -const mocks = await vi.hoisted(async () => (await import('@/__test__/providerTests')).makeProviderMocks()); +const mocks = await vi.hoisted(async () => (await import('../providerTests.ts')).makeProviderMocks()); vi.mock('@/providers/all', () => mocks); describe('ProviderControls.listSources()', () => { diff --git a/src/__test__/runner/meta.test.ts b/src/__test__/runner/meta.test.ts index 4db1fdc..59c3a3a 100644 --- a/src/__test__/runner/meta.test.ts +++ b/src/__test__/runner/meta.test.ts @@ -2,7 +2,7 @@ import { mockEmbeds, mockSources } from '@/__test__/providerTests'; import { makeProviders } from '@/main/builder'; import { afterEach, describe, expect, it, vi } from 'vitest'; -const mocks = await vi.hoisted(async () => (await import('@/__test__/providerTests')).makeProviderMocks()); +const mocks = await vi.hoisted(async () => (await import('../providerTests.ts')).makeProviderMocks()); vi.mock('@/providers/all', () => mocks); describe('ProviderControls.getMetadata()', () => { diff --git a/src/__test__/tsconfig.json b/src/__test__/tsconfig.json new file mode 100644 index 0000000..fc17519 --- /dev/null +++ b/src/__test__/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022"], + "module": "ES2022", + "declaration": true, + "outDir": "./lib", + "strict": true, + "moduleResolution": "NodeNext", + "allowImportingTsExtensions": true, + "noEmit": true, + "experimentalDecorators": true, + "isolatedModules": false, + "skipLibCheck": true, + "paths": { + "@/*": ["../*"], + "@entrypoint": ["../index.ts"] + } + }, + "include": ["./"] +} From 8df547fb1ddfd7e708aca163df898db6f111cc86 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 5 Sep 2023 21:30:10 +0200 Subject: [PATCH 14/23] update readme todos with descriptive todos --- README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 75c4979..167fa36 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,19 @@ features: > This package is still WIP Todos: - - add tests (integration, unit tests) + - add tests + - ProviderControls.runAll() + - are events called? + - custom source or embed order + - are fetchers called? + - is proxiedFetcher properly defaulted back to normal fetcher? + - makeStandardFetcher() + - do all parameters get passed to real fetch as expected? + - does serialisation work as expected? (formdata + json + string) + - do baseUrl settings work? + - does json responses get automatically parsed? + - makeFullUrl() + - do slashes get converted correctly? - running individual scrapers - finish fetchers: - make baseUrl param work @@ -18,6 +30,7 @@ Todos: - automatically parse json - error logging for failed scrapers - make the lib not compile into one file, keep dependency structure + - add all real providers Future todos: - docs: examples for nodejs + browser From 6c445c33cf420b22d6c382516eba2b196994e45d Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 5 Sep 2023 21:39:09 +0200 Subject: [PATCH 15/23] remove done todo --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 167fa36..c93054f 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,6 @@ Todos: - proper serialization (with content-type headers) for standard fetcher - automatically parse json - error logging for failed scrapers - - make the lib not compile into one file, keep dependency structure - add all real providers Future todos: From 26e8653ce4656798e57878b94f187b7f9d51e883 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 5 Sep 2023 22:18:59 +0200 Subject: [PATCH 16/23] Add documentation --- .docs/.gitignore | 3 + .docs/.vitepress/config.mts | 28 + .docs/package-lock.json | 1252 ++++++++++++++++++++++++++++++++ .docs/package.json | 9 + .docs/src/get-started/start.md | 49 ++ .docs/src/index.md | 24 + .docs/src/reference/start.md | 85 +++ docs/basic.js | 19 - 8 files changed, 1450 insertions(+), 19 deletions(-) create mode 100644 .docs/.gitignore create mode 100644 .docs/.vitepress/config.mts create mode 100644 .docs/package-lock.json create mode 100644 .docs/package.json create mode 100644 .docs/src/get-started/start.md create mode 100644 .docs/src/index.md create mode 100644 .docs/src/reference/start.md delete mode 100644 docs/basic.js diff --git a/.docs/.gitignore b/.docs/.gitignore new file mode 100644 index 0000000..5506568 --- /dev/null +++ b/.docs/.gitignore @@ -0,0 +1,3 @@ +node_modules +.vitepress/cache +.vitepress/dist diff --git a/.docs/.vitepress/config.mts b/.docs/.vitepress/config.mts new file mode 100644 index 0000000..0d05790 --- /dev/null +++ b/.docs/.vitepress/config.mts @@ -0,0 +1,28 @@ +import { defineConfig } from 'vitepress' + +export default defineConfig({ + title: "MW provider docs", + description: "Documentation for @movie-web/providers", + srcDir: "src", + themeConfig: { + nav: [ + { text: 'Home', link: '/' }, + { text: 'Get Started', link: '/get-started/start' }, + { text: 'Reference', link: '/reference/start' } + ], + + sidebar: [ + { + text: 'Examples', + items: [ + { text: 'Markdown Examples', link: '/markdown-examples' }, + { text: 'Runtime API Examples', link: '/api-examples' } + ] + } + ], + + socialLinks: [ + { icon: 'github', link: 'https://github.com/movie-web/providers' } + ] + } +}) diff --git a/.docs/package-lock.json b/.docs/package-lock.json new file mode 100644 index 0000000..dbec1db --- /dev/null +++ b/.docs/package-lock.json @@ -0,0 +1,1252 @@ +{ + "name": ".docs", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "devDependencies": { + "vitepress": "^1.0.0-rc.10" + } + }, + "node_modules/@algolia/autocomplete-core": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-core/-/autocomplete-core-1.9.3.tgz", + "integrity": "sha512-009HdfugtGCdC4JdXUbVJClA0q0zh24yyePn+KUGk3rP7j8FEe/m5Yo/z65gn6nP/cM39PxpzqKrL7A6fP6PPw==", + "dev": true, + "dependencies": { + "@algolia/autocomplete-plugin-algolia-insights": "1.9.3", + "@algolia/autocomplete-shared": "1.9.3" + } + }, + "node_modules/@algolia/autocomplete-plugin-algolia-insights": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-plugin-algolia-insights/-/autocomplete-plugin-algolia-insights-1.9.3.tgz", + "integrity": "sha512-a/yTUkcO/Vyy+JffmAnTWbr4/90cLzw+CC3bRbhnULr/EM0fGNvM13oQQ14f2moLMcVDyAx/leczLlAOovhSZg==", + "dev": true, + "dependencies": { + "@algolia/autocomplete-shared": "1.9.3" + }, + "peerDependencies": { + "search-insights": ">= 1 < 3" + } + }, + "node_modules/@algolia/autocomplete-preset-algolia": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-preset-algolia/-/autocomplete-preset-algolia-1.9.3.tgz", + "integrity": "sha512-d4qlt6YmrLMYy95n5TB52wtNDr6EgAIPH81dvvvW8UmuWRgxEtY0NJiPwl/h95JtG2vmRM804M0DSwMCNZlzRA==", + "dev": true, + "dependencies": { + "@algolia/autocomplete-shared": "1.9.3" + }, + "peerDependencies": { + "@algolia/client-search": ">= 4.9.1 < 6", + "algoliasearch": ">= 4.9.1 < 6" + } + }, + "node_modules/@algolia/autocomplete-shared": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-shared/-/autocomplete-shared-1.9.3.tgz", + "integrity": "sha512-Wnm9E4Ye6Rl6sTTqjoymD+l8DjSTHsHboVRYrKgEt8Q7UHm9nYbqhN/i0fhUYA3OAEH7WA8x3jfpnmJm3rKvaQ==", + "dev": true, + "peerDependencies": { + "@algolia/client-search": ">= 4.9.1 < 6", + "algoliasearch": ">= 4.9.1 < 6" + } + }, + "node_modules/@algolia/cache-browser-local-storage": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.19.1.tgz", + "integrity": "sha512-FYAZWcGsFTTaSAwj9Std8UML3Bu8dyWDncM7Ls8g+58UOe4XYdlgzXWbrIgjaguP63pCCbMoExKr61B+ztK3tw==", + "dev": true, + "dependencies": { + "@algolia/cache-common": "4.19.1" + } + }, + "node_modules/@algolia/cache-common": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/cache-common/-/cache-common-4.19.1.tgz", + "integrity": "sha512-XGghi3l0qA38HiqdoUY+wvGyBsGvKZ6U3vTiMBT4hArhP3fOGLXpIINgMiiGjTe4FVlTa5a/7Zf2bwlIHfRqqg==", + "dev": true + }, + "node_modules/@algolia/cache-in-memory": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/cache-in-memory/-/cache-in-memory-4.19.1.tgz", + "integrity": "sha512-+PDWL+XALGvIginigzu8oU6eWw+o76Z8zHbBovWYcrtWOEtinbl7a7UTt3x3lthv+wNuFr/YD1Gf+B+A9V8n5w==", + "dev": true, + "dependencies": { + "@algolia/cache-common": "4.19.1" + } + }, + "node_modules/@algolia/client-account": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/client-account/-/client-account-4.19.1.tgz", + "integrity": "sha512-Oy0ritA2k7AMxQ2JwNpfaEcgXEDgeyKu0V7E7xt/ZJRdXfEpZcwp9TOg4TJHC7Ia62gIeT2Y/ynzsxccPw92GA==", + "dev": true, + "dependencies": { + "@algolia/client-common": "4.19.1", + "@algolia/client-search": "4.19.1", + "@algolia/transporter": "4.19.1" + } + }, + "node_modules/@algolia/client-analytics": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/client-analytics/-/client-analytics-4.19.1.tgz", + "integrity": "sha512-5QCq2zmgdZLIQhHqwl55ZvKVpLM3DNWjFI4T+bHr3rGu23ew2bLO4YtyxaZeChmDb85jUdPDouDlCumGfk6wOg==", + "dev": true, + "dependencies": { + "@algolia/client-common": "4.19.1", + "@algolia/client-search": "4.19.1", + "@algolia/requester-common": "4.19.1", + "@algolia/transporter": "4.19.1" + } + }, + "node_modules/@algolia/client-common": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-4.19.1.tgz", + "integrity": "sha512-3kAIVqTcPrjfS389KQvKzliC559x+BDRxtWamVJt8IVp7LGnjq+aVAXg4Xogkur1MUrScTZ59/AaUd5EdpyXgA==", + "dev": true, + "dependencies": { + "@algolia/requester-common": "4.19.1", + "@algolia/transporter": "4.19.1" + } + }, + "node_modules/@algolia/client-personalization": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/client-personalization/-/client-personalization-4.19.1.tgz", + "integrity": "sha512-8CWz4/H5FA+krm9HMw2HUQenizC/DxUtsI5oYC0Jxxyce1vsr8cb1aEiSJArQT6IzMynrERif1RVWLac1m36xw==", + "dev": true, + "dependencies": { + "@algolia/client-common": "4.19.1", + "@algolia/requester-common": "4.19.1", + "@algolia/transporter": "4.19.1" + } + }, + "node_modules/@algolia/client-search": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-4.19.1.tgz", + "integrity": "sha512-mBecfMFS4N+yK/p0ZbK53vrZbL6OtWMk8YmnOv1i0LXx4pelY8TFhqKoTit3NPVPwoSNN0vdSN9dTu1xr1XOVw==", + "dev": true, + "dependencies": { + "@algolia/client-common": "4.19.1", + "@algolia/requester-common": "4.19.1", + "@algolia/transporter": "4.19.1" + } + }, + "node_modules/@algolia/logger-common": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/logger-common/-/logger-common-4.19.1.tgz", + "integrity": "sha512-i6pLPZW/+/YXKis8gpmSiNk1lOmYCmRI6+x6d2Qk1OdfvX051nRVdalRbEcVTpSQX6FQAoyeaui0cUfLYW5Elw==", + "dev": true + }, + "node_modules/@algolia/logger-console": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/logger-console/-/logger-console-4.19.1.tgz", + "integrity": "sha512-jj72k9GKb9W0c7TyC3cuZtTr0CngLBLmc8trzZlXdfvQiigpUdvTi1KoWIb2ZMcRBG7Tl8hSb81zEY3zI2RlXg==", + "dev": true, + "dependencies": { + "@algolia/logger-common": "4.19.1" + } + }, + "node_modules/@algolia/requester-browser-xhr": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-4.19.1.tgz", + "integrity": "sha512-09K/+t7lptsweRTueHnSnmPqIxbHMowejAkn9XIcJMLdseS3zl8ObnS5GWea86mu3vy4+8H+ZBKkUN82Zsq/zg==", + "dev": true, + "dependencies": { + "@algolia/requester-common": "4.19.1" + } + }, + "node_modules/@algolia/requester-common": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/requester-common/-/requester-common-4.19.1.tgz", + "integrity": "sha512-BisRkcWVxrDzF1YPhAckmi2CFYK+jdMT60q10d7z3PX+w6fPPukxHRnZwooiTUrzFe50UBmLItGizWHP5bDzVQ==", + "dev": true + }, + "node_modules/@algolia/requester-node-http": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-4.19.1.tgz", + "integrity": "sha512-6DK52DHviBHTG2BK/Vv2GIlEw7i+vxm7ypZW0Z7vybGCNDeWzADx+/TmxjkES2h15+FZOqVf/Ja677gePsVItA==", + "dev": true, + "dependencies": { + "@algolia/requester-common": "4.19.1" + } + }, + "node_modules/@algolia/transporter": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/transporter/-/transporter-4.19.1.tgz", + "integrity": "sha512-nkpvPWbpuzxo1flEYqNIbGz7xhfhGOKGAZS7tzC+TELgEmi7z99qRyTfNSUlW7LZmB3ACdnqAo+9A9KFBENviQ==", + "dev": true, + "dependencies": { + "@algolia/cache-common": "4.19.1", + "@algolia/logger-common": "4.19.1", + "@algolia/requester-common": "4.19.1" + } + }, + "node_modules/@babel/parser": { + "version": "7.22.15", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.22.15.tgz", + "integrity": "sha512-RWmQ/sklUN9BvGGpCDgSubhHWfAx24XDTDObup4ffvxaYsptOg2P3KG0j+1eWKLxpkX0j0uHxmpq2Z1SP/VhxA==", + "dev": true, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@docsearch/css": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/@docsearch/css/-/css-3.5.2.tgz", + "integrity": "sha512-SPiDHaWKQZpwR2siD0KQUwlStvIAnEyK6tAE2h2Wuoq8ue9skzhlyVQ1ddzOxX6khULnAALDiR/isSF3bnuciA==", + "dev": true + }, + "node_modules/@docsearch/js": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/@docsearch/js/-/js-3.5.2.tgz", + "integrity": "sha512-p1YFTCDflk8ieHgFJYfmyHBki1D61+U9idwrLh+GQQMrBSP3DLGKpy0XUJtPjAOPltcVbqsTjiPFfH7JImjUNg==", + "dev": true, + "dependencies": { + "@docsearch/react": "3.5.2", + "preact": "^10.0.0" + } + }, + "node_modules/@docsearch/react": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/@docsearch/react/-/react-3.5.2.tgz", + "integrity": "sha512-9Ahcrs5z2jq/DcAvYtvlqEBHImbm4YJI8M9y0x6Tqg598P40HTEkX7hsMcIuThI+hTFxRGZ9hll0Wygm2yEjng==", + "dev": true, + "dependencies": { + "@algolia/autocomplete-core": "1.9.3", + "@algolia/autocomplete-preset-algolia": "1.9.3", + "@docsearch/css": "3.5.2", + "algoliasearch": "^4.19.1" + }, + "peerDependencies": { + "@types/react": ">= 16.8.0 < 19.0.0", + "react": ">= 16.8.0 < 19.0.0", + "react-dom": ">= 16.8.0 < 19.0.0", + "search-insights": ">= 1 < 3" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "react": { + "optional": true + }, + "react-dom": { + "optional": true + }, + "search-insights": { + "optional": true + } + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.18.20.tgz", + "integrity": "sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.18.20.tgz", + "integrity": "sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.18.20.tgz", + "integrity": "sha512-8GDdlePJA8D6zlZYJV/jnrRAi6rOiNaCC/JclcXpB+KIuvfBN4owLtgzY2bsxnx666XjJx2kDPUmnTtR8qKQUg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.18.20.tgz", + "integrity": "sha512-bxRHW5kHU38zS2lPTPOyuyTm+S+eobPUnTNkdJEfAddYgEcll4xkT8DB9d2008DtTbl7uJag2HuE5NZAZgnNEA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.18.20.tgz", + "integrity": "sha512-pc5gxlMDxzm513qPGbCbDukOdsGtKhfxD1zJKXjCCcU7ju50O7MeAZ8c4krSJcOIJGFR+qx21yMMVYwiQvyTyQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.18.20.tgz", + "integrity": "sha512-yqDQHy4QHevpMAaxhhIwYPMv1NECwOvIpGCZkECn8w2WFHXjEwrBn3CeNIYsibZ/iZEUemj++M26W3cNR5h+Tw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.18.20.tgz", + "integrity": "sha512-tgWRPPuQsd3RmBZwarGVHZQvtzfEBOreNuxEMKFcd5DaDn2PbBxfwLcj4+aenoh7ctXcbXmOQIn8HI6mCSw5MQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.18.20.tgz", + "integrity": "sha512-/5bHkMWnq1EgKr1V+Ybz3s1hWXok7mDFUMQ4cG10AfW3wL02PSZi5kFpYKrptDsgb2WAJIvRcDm+qIvXf/apvg==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.18.20.tgz", + "integrity": "sha512-2YbscF+UL7SQAVIpnWvYwM+3LskyDmPhe31pE7/aoTMFKKzIc9lLbyGUpmmb8a8AixOL61sQ/mFh3jEjHYFvdA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.18.20.tgz", + "integrity": "sha512-P4etWwq6IsReT0E1KHU40bOnzMHoH73aXp96Fs8TIT6z9Hu8G6+0SHSw9i2isWrD2nbx2qo5yUqACgdfVGx7TA==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.18.20.tgz", + "integrity": "sha512-nXW8nqBTrOpDLPgPY9uV+/1DjxoQ7DoB2N8eocyq8I9XuqJ7BiAMDMf9n1xZM9TgW0J8zrquIb/A7s3BJv7rjg==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.18.20.tgz", + "integrity": "sha512-d5NeaXZcHp8PzYy5VnXV3VSd2D328Zb+9dEq5HE6bw6+N86JVPExrA6O68OPwobntbNJ0pzCpUFZTo3w0GyetQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.18.20.tgz", + "integrity": "sha512-WHPyeScRNcmANnLQkq6AfyXRFr5D6N2sKgkFo2FqguP44Nw2eyDlbTdZwd9GYk98DZG9QItIiTlFLHJHjxP3FA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.18.20.tgz", + "integrity": "sha512-WSxo6h5ecI5XH34KC7w5veNnKkju3zBRLEQNY7mv5mtBmrP/MjNBCAlsM2u5hDBlS3NGcTQpoBvRzqBcRtpq1A==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.18.20.tgz", + "integrity": "sha512-+8231GMs3mAEth6Ja1iK0a1sQ3ohfcpzpRLH8uuc5/KVDFneH6jtAJLFGafpzpMRO6DzJ6AvXKze9LfFMrIHVQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.18.20.tgz", + "integrity": "sha512-UYqiqemphJcNsFEskc73jQ7B9jgwjWrSayxawS6UVFZGWrAAtkzjxSqnoclCXxWtfwLdzU+vTpcNYhpn43uP1w==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.18.20.tgz", + "integrity": "sha512-iO1c++VP6xUBUmltHZoMtCUdPlnPGdBom6IrO4gyKPFFVBKioIImVooR5I83nTew5UOYrk3gIJhbZh8X44y06A==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.18.20.tgz", + "integrity": "sha512-e5e4YSsuQfX4cxcygw/UCPIEP6wbIL+se3sxPdCiMbFLBWu0eiZOJ7WoD+ptCLrmjZBK1Wk7I6D/I3NglUGOxg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.18.20.tgz", + "integrity": "sha512-kDbFRFp0YpTQVVrqUd5FTYmWo45zGaXe0X8E1G/LKFC0v8x0vWrhOWSLITcCn63lmZIxfOMXtCfti/RxN/0wnQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.18.20.tgz", + "integrity": "sha512-ddYFR6ItYgoaq4v4JmQQaAI5s7npztfV4Ag6NrhiaW0RrnOXqBkgwZLofVTlq1daVTQNhtI5oieTvkRPfZrePg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.18.20.tgz", + "integrity": "sha512-Wv7QBi3ID/rROT08SABTS7eV4hX26sVduqDOTe1MvGMjNd3EjOz4b7zeexIR62GTIEKrfJXKL9LFxTYgkyeu7g==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.18.20.tgz", + "integrity": "sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.4.15", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", + "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", + "dev": true + }, + "node_modules/@types/web-bluetooth": { + "version": "0.0.17", + "resolved": "https://registry.npmjs.org/@types/web-bluetooth/-/web-bluetooth-0.0.17.tgz", + "integrity": "sha512-4p9vcSmxAayx72yn70joFoL44c9MO/0+iVEBIQXe3v2h2SiAsEIo/G5v6ObFWvNKRFjbrVadNf9LqEEZeQPzdA==", + "dev": true + }, + "node_modules/@vue/compiler-core": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.3.4.tgz", + "integrity": "sha512-cquyDNvZ6jTbf/+x+AgM2Arrp6G4Dzbb0R64jiG804HRMfRiFXWI6kqUVqZ6ZR0bQhIoQjB4+2bhNtVwndW15g==", + "dev": true, + "dependencies": { + "@babel/parser": "^7.21.3", + "@vue/shared": "3.3.4", + "estree-walker": "^2.0.2", + "source-map-js": "^1.0.2" + } + }, + "node_modules/@vue/compiler-dom": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@vue/compiler-dom/-/compiler-dom-3.3.4.tgz", + "integrity": "sha512-wyM+OjOVpuUukIq6p5+nwHYtj9cFroz9cwkfmP9O1nzH68BenTTv0u7/ndggT8cIQlnBeOo6sUT/gvHcIkLA5w==", + "dev": true, + "dependencies": { + "@vue/compiler-core": "3.3.4", + "@vue/shared": "3.3.4" + } + }, + "node_modules/@vue/compiler-sfc": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-3.3.4.tgz", + "integrity": "sha512-6y/d8uw+5TkCuzBkgLS0v3lSM3hJDntFEiUORM11pQ/hKvkhSKZrXW6i69UyXlJQisJxuUEJKAWEqWbWsLeNKQ==", + "dev": true, + "dependencies": { + "@babel/parser": "^7.20.15", + "@vue/compiler-core": "3.3.4", + "@vue/compiler-dom": "3.3.4", + "@vue/compiler-ssr": "3.3.4", + "@vue/reactivity-transform": "3.3.4", + "@vue/shared": "3.3.4", + "estree-walker": "^2.0.2", + "magic-string": "^0.30.0", + "postcss": "^8.1.10", + "source-map-js": "^1.0.2" + } + }, + "node_modules/@vue/compiler-ssr": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@vue/compiler-ssr/-/compiler-ssr-3.3.4.tgz", + "integrity": "sha512-m0v6oKpup2nMSehwA6Uuu+j+wEwcy7QmwMkVNVfrV9P2qE5KshC6RwOCq8fjGS/Eak/uNb8AaWekfiXxbBB6gQ==", + "dev": true, + "dependencies": { + "@vue/compiler-dom": "3.3.4", + "@vue/shared": "3.3.4" + } + }, + "node_modules/@vue/devtools-api": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/@vue/devtools-api/-/devtools-api-6.5.0.tgz", + "integrity": "sha512-o9KfBeaBmCKl10usN4crU53fYtC1r7jJwdGKjPT24t348rHxgfpZ0xL3Xm/gLUYnc0oTp8LAmrxOeLyu6tbk2Q==", + "dev": true + }, + "node_modules/@vue/reactivity": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@vue/reactivity/-/reactivity-3.3.4.tgz", + "integrity": "sha512-kLTDLwd0B1jG08NBF3R5rqULtv/f8x3rOFByTDz4J53ttIQEDmALqKqXY0J+XQeN0aV2FBxY8nJDf88yvOPAqQ==", + "dev": true, + "dependencies": { + "@vue/shared": "3.3.4" + } + }, + "node_modules/@vue/reactivity-transform": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@vue/reactivity-transform/-/reactivity-transform-3.3.4.tgz", + "integrity": "sha512-MXgwjako4nu5WFLAjpBnCj/ieqcjE2aJBINUNQzkZQfzIZA4xn+0fV1tIYBJvvva3N3OvKGofRLvQIwEQPpaXw==", + "dev": true, + "dependencies": { + "@babel/parser": "^7.20.15", + "@vue/compiler-core": "3.3.4", + "@vue/shared": "3.3.4", + "estree-walker": "^2.0.2", + "magic-string": "^0.30.0" + } + }, + "node_modules/@vue/runtime-core": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@vue/runtime-core/-/runtime-core-3.3.4.tgz", + "integrity": "sha512-R+bqxMN6pWO7zGI4OMlmvePOdP2c93GsHFM/siJI7O2nxFRzj55pLwkpCedEY+bTMgp5miZ8CxfIZo3S+gFqvA==", + "dev": true, + "dependencies": { + "@vue/reactivity": "3.3.4", + "@vue/shared": "3.3.4" + } + }, + "node_modules/@vue/runtime-dom": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@vue/runtime-dom/-/runtime-dom-3.3.4.tgz", + "integrity": "sha512-Aj5bTJ3u5sFsUckRghsNjVTtxZQ1OyMWCr5dZRAPijF/0Vy4xEoRCwLyHXcj4D0UFbJ4lbx3gPTgg06K/GnPnQ==", + "dev": true, + "dependencies": { + "@vue/runtime-core": "3.3.4", + "@vue/shared": "3.3.4", + "csstype": "^3.1.1" + } + }, + "node_modules/@vue/server-renderer": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@vue/server-renderer/-/server-renderer-3.3.4.tgz", + "integrity": "sha512-Q6jDDzR23ViIb67v+vM1Dqntu+HUexQcsWKhhQa4ARVzxOY2HbC7QRW/ggkDBd5BU+uM1sV6XOAP0b216o34JQ==", + "dev": true, + "dependencies": { + "@vue/compiler-ssr": "3.3.4", + "@vue/shared": "3.3.4" + }, + "peerDependencies": { + "vue": "3.3.4" + } + }, + "node_modules/@vue/shared": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@vue/shared/-/shared-3.3.4.tgz", + "integrity": "sha512-7OjdcV8vQ74eiz1TZLzZP4JwqM5fA94K6yntPS5Z25r9HDuGNzaGdgvwKYq6S+MxwF0TFRwe50fIR/MYnakdkQ==", + "dev": true + }, + "node_modules/@vueuse/core": { + "version": "10.4.1", + "resolved": "https://registry.npmjs.org/@vueuse/core/-/core-10.4.1.tgz", + "integrity": "sha512-DkHIfMIoSIBjMgRRvdIvxsyboRZQmImofLyOHADqiVbQVilP8VVHDhBX2ZqoItOgu7dWa8oXiNnScOdPLhdEXg==", + "dev": true, + "dependencies": { + "@types/web-bluetooth": "^0.0.17", + "@vueuse/metadata": "10.4.1", + "@vueuse/shared": "10.4.1", + "vue-demi": ">=0.14.5" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@vueuse/core/node_modules/vue-demi": { + "version": "0.14.6", + "resolved": "https://registry.npmjs.org/vue-demi/-/vue-demi-0.14.6.tgz", + "integrity": "sha512-8QA7wrYSHKaYgUxDA5ZC24w+eHm3sYCbp0EzcDwKqN3p6HqtTCGR/GVsPyZW92unff4UlcSh++lmqDWN3ZIq4w==", + "dev": true, + "hasInstallScript": true, + "bin": { + "vue-demi-fix": "bin/vue-demi-fix.js", + "vue-demi-switch": "bin/vue-demi-switch.js" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + }, + "peerDependencies": { + "@vue/composition-api": "^1.0.0-rc.1", + "vue": "^3.0.0-0 || ^2.6.0" + }, + "peerDependenciesMeta": { + "@vue/composition-api": { + "optional": true + } + } + }, + "node_modules/@vueuse/integrations": { + "version": "10.4.1", + "resolved": "https://registry.npmjs.org/@vueuse/integrations/-/integrations-10.4.1.tgz", + "integrity": "sha512-uRBPyG5Lxoh1A/J+boiioPT3ELEAPEo4t8W6Mr4yTKIQBeW/FcbsotZNPr4k9uz+3QEksMmflWloS9wCnypM7g==", + "dev": true, + "dependencies": { + "@vueuse/core": "10.4.1", + "@vueuse/shared": "10.4.1", + "vue-demi": ">=0.14.5" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + }, + "peerDependencies": { + "async-validator": "*", + "axios": "*", + "change-case": "*", + "drauu": "*", + "focus-trap": "*", + "fuse.js": "*", + "idb-keyval": "*", + "jwt-decode": "*", + "nprogress": "*", + "qrcode": "*", + "sortablejs": "*", + "universal-cookie": "*" + }, + "peerDependenciesMeta": { + "async-validator": { + "optional": true + }, + "axios": { + "optional": true + }, + "change-case": { + "optional": true + }, + "drauu": { + "optional": true + }, + "focus-trap": { + "optional": true + }, + "fuse.js": { + "optional": true + }, + "idb-keyval": { + "optional": true + }, + "jwt-decode": { + "optional": true + }, + "nprogress": { + "optional": true + }, + "qrcode": { + "optional": true + }, + "sortablejs": { + "optional": true + }, + "universal-cookie": { + "optional": true + } + } + }, + "node_modules/@vueuse/integrations/node_modules/vue-demi": { + "version": "0.14.6", + "resolved": "https://registry.npmjs.org/vue-demi/-/vue-demi-0.14.6.tgz", + "integrity": "sha512-8QA7wrYSHKaYgUxDA5ZC24w+eHm3sYCbp0EzcDwKqN3p6HqtTCGR/GVsPyZW92unff4UlcSh++lmqDWN3ZIq4w==", + "dev": true, + "hasInstallScript": true, + "bin": { + "vue-demi-fix": "bin/vue-demi-fix.js", + "vue-demi-switch": "bin/vue-demi-switch.js" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + }, + "peerDependencies": { + "@vue/composition-api": "^1.0.0-rc.1", + "vue": "^3.0.0-0 || ^2.6.0" + }, + "peerDependenciesMeta": { + "@vue/composition-api": { + "optional": true + } + } + }, + "node_modules/@vueuse/metadata": { + "version": "10.4.1", + "resolved": "https://registry.npmjs.org/@vueuse/metadata/-/metadata-10.4.1.tgz", + "integrity": "sha512-2Sc8X+iVzeuMGHr6O2j4gv/zxvQGGOYETYXEc41h0iZXIRnRbJZGmY/QP8dvzqUelf8vg0p/yEA5VpCEu+WpZg==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@vueuse/shared": { + "version": "10.4.1", + "resolved": "https://registry.npmjs.org/@vueuse/shared/-/shared-10.4.1.tgz", + "integrity": "sha512-vz5hbAM4qA0lDKmcr2y3pPdU+2EVw/yzfRsBdu+6+USGa4PxqSQRYIUC9/NcT06y+ZgaTsyURw2I9qOFaaXHAg==", + "dev": true, + "dependencies": { + "vue-demi": ">=0.14.5" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@vueuse/shared/node_modules/vue-demi": { + "version": "0.14.6", + "resolved": "https://registry.npmjs.org/vue-demi/-/vue-demi-0.14.6.tgz", + "integrity": "sha512-8QA7wrYSHKaYgUxDA5ZC24w+eHm3sYCbp0EzcDwKqN3p6HqtTCGR/GVsPyZW92unff4UlcSh++lmqDWN3ZIq4w==", + "dev": true, + "hasInstallScript": true, + "bin": { + "vue-demi-fix": "bin/vue-demi-fix.js", + "vue-demi-switch": "bin/vue-demi-switch.js" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + }, + "peerDependencies": { + "@vue/composition-api": "^1.0.0-rc.1", + "vue": "^3.0.0-0 || ^2.6.0" + }, + "peerDependenciesMeta": { + "@vue/composition-api": { + "optional": true + } + } + }, + "node_modules/algoliasearch": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-4.19.1.tgz", + "integrity": "sha512-IJF5b93b2MgAzcE/tuzW0yOPnuUyRgGAtaPv5UUywXM8kzqfdwZTO4sPJBzoGz1eOy6H9uEchsJsBFTELZSu+g==", + "dev": true, + "dependencies": { + "@algolia/cache-browser-local-storage": "4.19.1", + "@algolia/cache-common": "4.19.1", + "@algolia/cache-in-memory": "4.19.1", + "@algolia/client-account": "4.19.1", + "@algolia/client-analytics": "4.19.1", + "@algolia/client-common": "4.19.1", + "@algolia/client-personalization": "4.19.1", + "@algolia/client-search": "4.19.1", + "@algolia/logger-common": "4.19.1", + "@algolia/logger-console": "4.19.1", + "@algolia/requester-browser-xhr": "4.19.1", + "@algolia/requester-common": "4.19.1", + "@algolia/requester-node-http": "4.19.1", + "@algolia/transporter": "4.19.1" + } + }, + "node_modules/ansi-sequence-parser": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ansi-sequence-parser/-/ansi-sequence-parser-1.1.1.tgz", + "integrity": "sha512-vJXt3yiaUL4UU546s3rPXlsry/RnM730G1+HkpKE012AN0sx1eOrxSu95oKDIonskeLTijMgqWZ3uDEe3NFvyg==", + "dev": true + }, + "node_modules/csstype": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.2.tgz", + "integrity": "sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ==", + "dev": true + }, + "node_modules/esbuild": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.18.20.tgz", + "integrity": "sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/android-arm": "0.18.20", + "@esbuild/android-arm64": "0.18.20", + "@esbuild/android-x64": "0.18.20", + "@esbuild/darwin-arm64": "0.18.20", + "@esbuild/darwin-x64": "0.18.20", + "@esbuild/freebsd-arm64": "0.18.20", + "@esbuild/freebsd-x64": "0.18.20", + "@esbuild/linux-arm": "0.18.20", + "@esbuild/linux-arm64": "0.18.20", + "@esbuild/linux-ia32": "0.18.20", + "@esbuild/linux-loong64": "0.18.20", + "@esbuild/linux-mips64el": "0.18.20", + "@esbuild/linux-ppc64": "0.18.20", + "@esbuild/linux-riscv64": "0.18.20", + "@esbuild/linux-s390x": "0.18.20", + "@esbuild/linux-x64": "0.18.20", + "@esbuild/netbsd-x64": "0.18.20", + "@esbuild/openbsd-x64": "0.18.20", + "@esbuild/sunos-x64": "0.18.20", + "@esbuild/win32-arm64": "0.18.20", + "@esbuild/win32-ia32": "0.18.20", + "@esbuild/win32-x64": "0.18.20" + } + }, + "node_modules/estree-walker": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", + "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", + "dev": true + }, + "node_modules/focus-trap": { + "version": "7.5.2", + "resolved": "https://registry.npmjs.org/focus-trap/-/focus-trap-7.5.2.tgz", + "integrity": "sha512-p6vGNNWLDGwJCiEjkSK6oERj/hEyI9ITsSwIUICBoKLlWiTWXJRfQibCwcoi50rTZdbi87qDtUlMCmQwsGSgPw==", + "dev": true, + "dependencies": { + "tabbable": "^6.2.0" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/jsonc-parser": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.2.0.tgz", + "integrity": "sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w==", + "dev": true + }, + "node_modules/magic-string": { + "version": "0.30.3", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.3.tgz", + "integrity": "sha512-B7xGbll2fG/VjP+SWg4sX3JynwIU0mjoTc6MPpKNuIvftk6u6vqhDnk1R80b8C2GBR6ywqy+1DcKBrevBg+bmw==", + "dev": true, + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.4.15" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/mark.js": { + "version": "8.11.1", + "resolved": "https://registry.npmjs.org/mark.js/-/mark.js-8.11.1.tgz", + "integrity": "sha512-1I+1qpDt4idfgLQG+BNWmrqku+7/2bi5nLf4YwF8y8zXvmfiTBY3PV3ZibfrjBueCByROpuBjLLFCajqkgYoLQ==", + "dev": true + }, + "node_modules/minisearch": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/minisearch/-/minisearch-6.1.0.tgz", + "integrity": "sha512-PNxA/X8pWk+TiqPbsoIYH0GQ5Di7m6326/lwU/S4mlo4wGQddIcf/V//1f9TB0V4j59b57b+HZxt8h3iMROGvg==", + "dev": true + }, + "node_modules/nanoid": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", + "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/picocolors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", + "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", + "dev": true + }, + "node_modules/postcss": { + "version": "8.4.29", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.29.tgz", + "integrity": "sha512-cbI+jaqIeu/VGqXEarWkRCCffhjgXc0qjBtXpqJhTBohMUjUQnbBr0xqX3vEKudc4iviTewcJo5ajcec5+wdJw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "nanoid": "^3.3.6", + "picocolors": "^1.0.0", + "source-map-js": "^1.0.2" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/preact": { + "version": "10.17.1", + "resolved": "https://registry.npmjs.org/preact/-/preact-10.17.1.tgz", + "integrity": "sha512-X9BODrvQ4Ekwv9GURm9AKAGaomqXmip7NQTZgY7gcNmr7XE83adOMJvd3N42id1tMFU7ojiynRsYnY6/BRFxLA==", + "dev": true, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/preact" + } + }, + "node_modules/rollup": { + "version": "3.28.1", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-3.28.1.tgz", + "integrity": "sha512-R9OMQmIHJm9znrU3m3cpE8uhN0fGdXiawME7aZIpQqvpS/85+Vt1Hq1/yVIcYfOmaQiHjvXkQAoJukvLpau6Yw==", + "dev": true, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=14.18.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/search-insights": { + "version": "2.8.2", + "resolved": "https://registry.npmjs.org/search-insights/-/search-insights-2.8.2.tgz", + "integrity": "sha512-PxA9M5Q2bpBelVvJ3oDZR8nuY00Z6qwOxL53wNpgzV28M/D6u9WUbImDckjLSILBF8F1hn/mgyuUaOPtjow4Qw==", + "dev": true, + "peer": true + }, + "node_modules/shiki": { + "version": "0.14.4", + "resolved": "https://registry.npmjs.org/shiki/-/shiki-0.14.4.tgz", + "integrity": "sha512-IXCRip2IQzKwxArNNq1S+On4KPML3Yyn8Zzs/xRgcgOWIr8ntIK3IKzjFPfjy/7kt9ZMjc+FItfqHRBg8b6tNQ==", + "dev": true, + "dependencies": { + "ansi-sequence-parser": "^1.1.0", + "jsonc-parser": "^3.2.0", + "vscode-oniguruma": "^1.7.0", + "vscode-textmate": "^8.0.0" + } + }, + "node_modules/source-map-js": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", + "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tabbable": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/tabbable/-/tabbable-6.2.0.tgz", + "integrity": "sha512-Cat63mxsVJlzYvN51JmVXIgNoUokrIaT2zLclCXjRd8boZ0004U4KCs/sToJ75C6sdlByWxpYnb5Boif1VSFew==", + "dev": true + }, + "node_modules/vite": { + "version": "4.4.9", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.4.9.tgz", + "integrity": "sha512-2mbUn2LlUmNASWwSCNSJ/EG2HuSRTnVNaydp6vMCm5VIqJsjMfbIWtbH2kDuwUVW5mMUKKZvGPX/rqeqVvv1XA==", + "dev": true, + "dependencies": { + "esbuild": "^0.18.10", + "postcss": "^8.4.27", + "rollup": "^3.27.1" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + }, + "peerDependencies": { + "@types/node": ">= 14", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.4.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + } + } + }, + "node_modules/vitepress": { + "version": "1.0.0-rc.10", + "resolved": "https://registry.npmjs.org/vitepress/-/vitepress-1.0.0-rc.10.tgz", + "integrity": "sha512-+MsahIWqq5WUEmj6MR4obcKYbT7im07jZPCQPdNJExkeOSbOAJ4xypSLx88x7rvtzWHhHc5aXbOhCRvGEGjFrw==", + "dev": true, + "dependencies": { + "@docsearch/css": "^3.5.2", + "@docsearch/js": "^3.5.2", + "@vue/devtools-api": "^6.5.0", + "@vueuse/core": "^10.4.1", + "@vueuse/integrations": "^10.4.1", + "focus-trap": "^7.5.2", + "mark.js": "8.11.1", + "minisearch": "^6.1.0", + "shiki": "^0.14.3", + "vite": "^4.4.9", + "vue": "^3.3.4" + }, + "bin": { + "vitepress": "bin/vitepress.js" + } + }, + "node_modules/vscode-oniguruma": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-1.7.0.tgz", + "integrity": "sha512-L9WMGRfrjOhgHSdOYgCt/yRMsXzLDJSL7BPrOZt73gU0iWO4mpqzqQzOz5srxqTvMBaR0XZTSrVWo4j55Rc6cA==", + "dev": true + }, + "node_modules/vscode-textmate": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-8.0.0.tgz", + "integrity": "sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==", + "dev": true + }, + "node_modules/vue": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/vue/-/vue-3.3.4.tgz", + "integrity": "sha512-VTyEYn3yvIeY1Py0WaYGZsXnz3y5UnGi62GjVEqvEGPl6nxbOrCXbVOTQWBEJUqAyTUk2uJ5JLVnYJ6ZzGbrSw==", + "dev": true, + "dependencies": { + "@vue/compiler-dom": "3.3.4", + "@vue/compiler-sfc": "3.3.4", + "@vue/runtime-dom": "3.3.4", + "@vue/server-renderer": "3.3.4", + "@vue/shared": "3.3.4" + } + } + } +} diff --git a/.docs/package.json b/.docs/package.json new file mode 100644 index 0000000..7e60560 --- /dev/null +++ b/.docs/package.json @@ -0,0 +1,9 @@ +{ + "scripts": { + "dev": "vitepress dev .", + "build": "vitepress build ." + }, + "devDependencies": { + "vitepress": "^1.0.0-rc.10" + } +} diff --git a/.docs/src/get-started/start.md b/.docs/src/get-started/start.md new file mode 100644 index 0000000..6bd8bb5 --- /dev/null +++ b/.docs/src/get-started/start.md @@ -0,0 +1,49 @@ +--- +outline: deep +--- + +# Runtime API Examples + +This page demonstrates usage of some of the runtime APIs provided by VitePress. + +The main `useData()` API can be used to access site, theme, and page data for the current page. It works in both `.md` and `.vue` files: + +```md + + +## Results + +### Theme Data +
{{ theme }}
+ +### Page Data +
{{ page }}
+ +### Page Frontmatter +
{{ frontmatter }}
+``` + + + +## Results + +### Theme Data +
{{ theme }}
+ +### Page Data +
{{ page }}
+ +### Page Frontmatter +
{{ frontmatter }}
+ +## More + +Check out the documentation for the [full list of runtime APIs](https://vitepress.dev/reference/runtime-api#usedata). diff --git a/.docs/src/index.md b/.docs/src/index.md new file mode 100644 index 0000000..f1c7ae4 --- /dev/null +++ b/.docs/src/index.md @@ -0,0 +1,24 @@ +--- +layout: home + +hero: + name: "@movie-web/providers" + tagline: Providers for all kinds of media + actions: + - theme: brand + text: Get Started + link: /get-started/start + - theme: alt + text: reference + link: /reference/start + +features: + - title: All the scraping! + icon: '!' + details: scrape popular streaming websites + - title: Client & server + icon: '!' + details: This library can be ran both server-side and client-side (with CORS proxy) +--- + + diff --git a/.docs/src/reference/start.md b/.docs/src/reference/start.md new file mode 100644 index 0000000..8e55eb8 --- /dev/null +++ b/.docs/src/reference/start.md @@ -0,0 +1,85 @@ +# Markdown Extension Examples + +This page demonstrates some of the built-in markdown extensions provided by VitePress. + +## Syntax Highlighting + +VitePress provides Syntax Highlighting powered by [Shiki](https://github.com/shikijs/shiki), with additional features like line-highlighting: + +**Input** + +```` +```js{4} +export default { + data () { + return { + msg: 'Highlighted!' + } + } +} +``` +```` + +**Output** + +```js{4} +export default { + data () { + return { + msg: 'Highlighted!' + } + } +} +``` + +## Custom Containers + +**Input** + +```md +::: info +This is an info box. +::: + +::: tip +This is a tip. +::: + +::: warning +This is a warning. +::: + +::: danger +This is a dangerous warning. +::: + +::: details +This is a details block. +::: +``` + +**Output** + +::: info +This is an info box. +::: + +::: tip +This is a tip. +::: + +::: warning +This is a warning. +::: + +::: danger +This is a dangerous warning. +::: + +::: details +This is a details block. +::: + +## More + +Check out the documentation for the [full list of markdown extensions](https://vitepress.dev/guide/markdown). diff --git a/docs/basic.js b/docs/basic.js deleted file mode 100644 index 7acc249..0000000 --- a/docs/basic.js +++ /dev/null @@ -1,19 +0,0 @@ -async function example() { - const providers = makeProviders({ - fetcher: makeStandardFetcher(fetch), - }); - - const source = await providers.runAll({ - media: { - title: 'Spider-Man: Across the Spider-Verse', - releaseYear: 2023, - imbdId: 'tt9362722', - tmdbId: '569094', - type: 'movie', - }, - }); - - if (!source) throw new Error("Couldn't find a stream"); - if (source.stream.type === 'file') return source.stream.qualities['1080']?.url; - if (source.stream.type === 'hls') return source.stream.playlist; -} From 461b39e2f1a2a3d0c0a1529dbb967b5a47c910c2 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 5 Sep 2023 22:24:45 +0200 Subject: [PATCH 17/23] doc deployment action --- .github/workflows/docs.yml | 43 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/docs.yml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..3cb6d14 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,43 @@ +name: Publish docs + +on: + push: + branches: + - master + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Install Node.js + uses: actions/setup-node@v3 + with: + node-version: 18 + + - name: Install packages + run: cd .docs && npm ci + + - name: Build + run: cd .docs && npm run build + + - name: Upload + uses: actions/upload-pages-artifact@v2 + with: + path: ./.docs/.vitepress/dist + + deploy: + needs: build + permissions: + pages: write + id-token: write + environment: + name: docs + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v2 From afa73466010809bb6b7a6b19adac57c6afab8b16 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 6 Sep 2023 14:30:21 +0200 Subject: [PATCH 18/23] add support for full url merging --- examples/.gitkeep | 0 src/__test__/fetchers/common.test.ts | 48 ++++++++++++++++++++++++++++ src/fetchers/common.ts | 18 +++++++++-- 3 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 examples/.gitkeep create mode 100644 src/__test__/fetchers/common.test.ts diff --git a/examples/.gitkeep b/examples/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/src/__test__/fetchers/common.test.ts b/src/__test__/fetchers/common.test.ts new file mode 100644 index 0000000..c067491 --- /dev/null +++ b/src/__test__/fetchers/common.test.ts @@ -0,0 +1,48 @@ +import { makeFullUrl } from "@/fetchers/common"; +import { describe, expect, it } from "vitest"; + +describe("makeFullUrl()", () => { + it('should pass normal url if no options', () => { + expect(makeFullUrl('https://example.com/hello/world')).toEqual("https://example.com/hello/world") + expect(makeFullUrl('https://example.com/hello/world?a=b')).toEqual("https://example.com/hello/world?a=b") + expect(makeFullUrl('https://example.com/hello/world?a=b#hello')).toEqual("https://example.com/hello/world?a=b#hello") + expect(makeFullUrl('https://example.com/hello/world#hello')).toEqual("https://example.com/hello/world#hello") + }) + + it('should append baseurl correctly', () => { + const correctResult = "https://example.com/hello/world"; + expect(makeFullUrl(correctResult, { baseUrl: '' })).toEqual(correctResult) + expect(makeFullUrl('/hello/world', { baseUrl: 'https://example.com' })).toEqual(correctResult) + expect(makeFullUrl('/hello/world', { baseUrl: 'https://example.com/' })).toEqual(correctResult) + expect(makeFullUrl('hello/world', { baseUrl: 'https://example.com/' })).toEqual(correctResult) + expect(makeFullUrl('hello/world', { baseUrl: 'https://example.com' })).toEqual(correctResult) + expect(makeFullUrl('/world', { baseUrl: 'https://example.com/hello' })).toEqual(correctResult) + expect(makeFullUrl('/world', { baseUrl: 'https://example.com/hello/' })).toEqual(correctResult) + expect(makeFullUrl('world', { baseUrl: 'https://example.com/hello/' })).toEqual(correctResult) + expect(makeFullUrl('world', { baseUrl: 'https://example.com/hello' })).toEqual(correctResult) + expect(makeFullUrl('world?a=b', { baseUrl: 'https://example.com/hello' })).toEqual("https://example.com/hello/world?a=b") + }) + + it('should throw with invalid baseurl combinations', () => { + expect(() => makeFullUrl('example.com/hello/world', { baseUrl: '' })).toThrowError() + expect(() => makeFullUrl('/hello/world', { baseUrl: 'example.com' })).toThrowError() + expect(() => makeFullUrl('/hello/world', { baseUrl: 'tcp://example.com' })).toThrowError() + expect(() => makeFullUrl('/hello/world', { baseUrl: 'tcp://example.com' })).toThrowError() + }) + + it('should add/merge query parameters', () => { + expect(makeFullUrl('https://example.com/hello/world', { query: { a: 'b' } })).toEqual("https://example.com/hello/world?a=b") + expect(makeFullUrl('https://example.com/hello/world/', { query: { a: 'b' } })).toEqual("https://example.com/hello/world/?a=b") + expect(makeFullUrl('https://example.com', { query: { a: 'b' } })).toEqual("https://example.com/?a=b") + expect(makeFullUrl('https://example.com/', { query: { a: 'b' } })).toEqual("https://example.com/?a=b") + + expect(makeFullUrl('https://example.com/hello/world?c=d', { query: { a: 'b' } })).toEqual("https://example.com/hello/world?c=d&a=b") + expect(makeFullUrl('https://example.com/hello/world?c=d', { query: {} })).toEqual("https://example.com/hello/world?c=d") + expect(makeFullUrl('https://example.com/hello/world?c=d')).toEqual("https://example.com/hello/world?c=d") + expect(makeFullUrl('https://example.com/hello/world?c=d', {})).toEqual("https://example.com/hello/world?c=d") + }) + + it('should work with a mix of multiple options', () => { + expect(makeFullUrl('/hello/world?c=d', { baseUrl: 'https://example.com/', query: { a: 'b' } })).toEqual("https://example.com/hello/world?c=d&a=b") + }) +}) diff --git a/src/fetchers/common.ts b/src/fetchers/common.ts index e0c1780..e31b6d1 100644 --- a/src/fetchers/common.ts +++ b/src/fetchers/common.ts @@ -1,10 +1,22 @@ import { Fetcher, FetcherOptions, UseableFetcher } from '@/fetchers/types'; +export type FullUrlOptions = Pick; + // make url with query params and base url used correctly -export function makeFullUrl(url: string, ops?: FetcherOptions): string { +export function makeFullUrl(url: string, ops?: FullUrlOptions): string { // glue baseUrl and rest of url together - const fullUrl = ops?.baseUrl ?? ''; - // TODO make full url + let leftSide = ops?.baseUrl ?? ''; + let rightSide = url; + + // left side should always end with slash, if its set + if (leftSide.length > 0 && !leftSide.endsWith('/')) leftSide += '/'; + + // right side should never start with slash + if (rightSide.startsWith('/')) rightSide = rightSide.slice(1); + + const fullUrl = leftSide + rightSide; + if (!fullUrl.startsWith('http://') && !fullUrl.startsWith('https://')) + throw new Error(`Invald URL -- URL doesn't start with a http scheme: '${fullUrl}'`); const parsedUrl = new URL(fullUrl); Object.entries(ops?.query ?? {}).forEach(([k, v]) => { From ec3efbd34424cd241caed752a2a2a029b83f5a8e Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 6 Sep 2023 14:32:13 +0200 Subject: [PATCH 19/23] add test coverage util --- .gitignore | 1 + package-lock.json | 360 ++++++++++++++++++++++++++++++++-------------- package.json | 4 +- 3 files changed, 256 insertions(+), 109 deletions(-) diff --git a/.gitignore b/.gitignore index d9baaa6..7525992 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules/ /lib +coverage diff --git a/package-lock.json b/package-lock.json index 014124f..0958f93 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "@types/crypto-js": "^4.1.1", "@typescript-eslint/eslint-plugin": "^5.60.0", "@typescript-eslint/parser": "^5.60.0", + "@vitest/coverage-v8": "^0.34.3", "eslint": "^8.30.0", "eslint-config-airbnb-base": "^15.0.0", "eslint-config-prettier": "^8.5.0", @@ -33,6 +34,19 @@ "vitest": "^0.32.2" } }, + "node_modules/@ampproject/remapping": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.1.tgz", + "integrity": "sha512-lFMjJTrFL3j7L9yBxwYfCq2k6qqwHyzuUl/XBnif78PWTJYyL/dfowQHWE3sp6U6ZzqWiiIZnpTMO96zhkjwtg==", + "dev": true, + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.0", + "@jridgewell/trace-mapping": "^0.3.9" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/@babel/parser": { "version": "7.22.5", "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.22.5.tgz", @@ -45,6 +59,12 @@ "node": ">=6.0.0" } }, + "node_modules/@bcoe/v8-coverage": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", + "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", + "dev": true + }, "node_modules/@esbuild/android-arm": { "version": "0.17.19", "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.17.19.tgz", @@ -486,12 +506,63 @@ "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", "dev": true }, + "node_modules/@istanbuljs/schema": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", + "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz", + "integrity": "sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==", + "dev": true, + "dependencies": { + "@jridgewell/set-array": "^1.0.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.9" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz", + "integrity": "sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/set-array": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", + "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/@jridgewell/sourcemap-codec": { "version": "1.4.15", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", "dev": true }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.19", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.19.tgz", + "integrity": "sha512-kf37QtfW+Hwx/buWGMPcR60iF9ziHa6r/CZJIHbmcm4+0qrXiVdxegAH0F6yddEVQ7zdkjcGCgCzUu+BcbhQxw==", + "dev": true, + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, "node_modules/@microsoft/api-extractor": { "version": "7.36.0", "resolved": "https://registry.npmjs.org/@microsoft/api-extractor/-/api-extractor-7.36.0.tgz", @@ -526,18 +597,6 @@ "@rushstack/node-core-library": "3.59.4" } }, - "node_modules/@microsoft/api-extractor/node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/@microsoft/api-extractor/node_modules/semver": { "version": "7.3.8", "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", @@ -566,12 +625,6 @@ "node": ">=12.20" } }, - "node_modules/@microsoft/api-extractor/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true - }, "node_modules/@microsoft/tsdoc": { "version": "0.14.2", "resolved": "https://registry.npmjs.org/@microsoft/tsdoc/-/tsdoc-0.14.2.tgz", @@ -723,18 +776,6 @@ "graceful-fs": "^4.1.6" } }, - "node_modules/@rushstack/node-core-library/node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/@rushstack/node-core-library/node_modules/semver": { "version": "7.3.8", "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", @@ -759,12 +800,6 @@ "node": ">= 4.0.0" } }, - "node_modules/@rushstack/node-core-library/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true - }, "node_modules/@rushstack/rig-package": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/@rushstack/rig-package/-/rig-package-0.4.0.tgz", @@ -901,6 +936,12 @@ "integrity": "sha512-LG4opVs2ANWZ1TJoKc937iMmNstM/d0ae1vNbnBvBhqCSezgVUOzcLCqbI5elV8Vy6WKwKjaqR+zO9VKirBBCA==", "dev": true }, + "node_modules/@types/istanbul-lib-coverage": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.4.tgz", + "integrity": "sha512-z/QT1XN4K4KYuslS23k62yDIDLwLFkzxOuMplDtObz0+y7VqJCaO2o+SPwHCvLFZh7xazvvoor2tA/hPz9ee7g==", + "dev": true + }, "node_modules/@types/json-schema": { "version": "7.0.12", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.12.tgz", @@ -959,18 +1000,6 @@ } } }, - "node_modules/@typescript-eslint/eslint-plugin/node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/@typescript-eslint/eslint-plugin/node_modules/semver": { "version": "7.5.2", "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.2.tgz", @@ -1001,12 +1030,6 @@ "typescript": ">=2.8.0 || >= 3.2.0-dev || >= 3.3.0-dev || >= 3.4.0-dev || >= 3.5.0-dev || >= 3.6.0-dev || >= 3.6.0-beta || >= 3.7.0-dev || >= 3.7.0-beta" } }, - "node_modules/@typescript-eslint/eslint-plugin/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true - }, "node_modules/@typescript-eslint/parser": { "version": "5.60.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.60.0.tgz", @@ -1133,18 +1156,6 @@ } } }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/@typescript-eslint/typescript-estree/node_modules/semver": { "version": "7.5.2", "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.2.tgz", @@ -1175,12 +1186,6 @@ "typescript": ">=2.8.0 || >= 3.2.0-dev || >= 3.3.0-dev || >= 3.4.0-dev || >= 3.5.0-dev || >= 3.6.0-dev || >= 3.6.0-beta || >= 3.7.0-dev || >= 3.7.0-beta" } }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true - }, "node_modules/@typescript-eslint/utils": { "version": "5.60.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-5.60.0.tgz", @@ -1229,18 +1234,6 @@ "node": ">=4.0" } }, - "node_modules/@typescript-eslint/utils/node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/@typescript-eslint/utils/node_modules/semver": { "version": "7.5.2", "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.2.tgz", @@ -1256,12 +1249,6 @@ "node": ">=10" } }, - "node_modules/@typescript-eslint/utils/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true - }, "node_modules/@typescript-eslint/visitor-keys": { "version": "5.60.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.60.0.tgz", @@ -1279,6 +1266,43 @@ "url": "https://opencollective.com/typescript-eslint" } }, + "node_modules/@vitest/coverage-v8": { + "version": "0.34.3", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-0.34.3.tgz", + "integrity": "sha512-bNjP0RHe8UxdklCigZlk6FVCNbOiqVjWnpZJ1zKixpvb7YHSaZiN/w+mzpvXIoqyxyePzKC+L+G1oj7SB20PJw==", + "dev": true, + "dependencies": { + "@ampproject/remapping": "^2.2.1", + "@bcoe/v8-coverage": "^0.2.3", + "istanbul-lib-coverage": "^3.2.0", + "istanbul-lib-report": "^3.0.1", + "istanbul-lib-source-maps": "^4.0.1", + "istanbul-reports": "^3.1.5", + "magic-string": "^0.30.1", + "picocolors": "^1.0.0", + "std-env": "^3.3.3", + "test-exclude": "^6.0.0", + "v8-to-istanbul": "^9.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "vitest": ">=0.32.0 <1" + } + }, + "node_modules/@vitest/coverage-v8/node_modules/magic-string": { + "version": "0.30.3", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.3.tgz", + "integrity": "sha512-B7xGbll2fG/VjP+SWg4sX3JynwIU0mjoTc6MPpKNuIvftk6u6vqhDnk1R80b8C2GBR6ywqy+1DcKBrevBg+bmw==", + "dev": true, + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.4.15" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/@vitest/expect": { "version": "0.32.2", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-0.32.2.tgz", @@ -1968,18 +1992,6 @@ "node": ">=10.18.0 <11 || >=12.14.0 <13 || >=14" } }, - "node_modules/concordance/node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/concordance/node_modules/semver": { "version": "7.5.2", "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.2.tgz", @@ -1995,18 +2007,18 @@ "node": ">=10" } }, - "node_modules/concordance/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true - }, "node_modules/confusing-browser-globals": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/confusing-browser-globals/-/confusing-browser-globals-1.0.11.tgz", "integrity": "sha512-JsPKdmh8ZkmnHxDk55FZ1TqVLvEQTvoByJZRN9jzI0UjxK/QgAmsphz7PGtqgPieQZ/CQcHWXCR7ATDNhGe+YA==", "dev": true }, + "node_modules/convert-source-map": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz", + "integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==", + "dev": true + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -3599,6 +3611,12 @@ "node": ">=10" } }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true + }, "node_modules/htmlparser2": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", @@ -4078,6 +4096,56 @@ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "dev": true }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.0.tgz", + "integrity": "sha512-eOeJ5BHCmHYvQK7xt9GkdHuzuCGS1Y6g9Gvnx3Ym33fz/HpLRYxiS0wHNr+m/MBC8B647Xt608vCDEvhl9c6Mw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "dev": true, + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-source-maps": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz", + "integrity": "sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==", + "dev": true, + "dependencies": { + "debug": "^4.1.1", + "istanbul-lib-coverage": "^3.0.0", + "source-map": "^0.6.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-reports": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.6.tgz", + "integrity": "sha512-TLgnMkKg3iTDsQ9PbPTdpfAK2DzjF9mqUG7RMgcQl8oFjad8ob4laGxv5XV5U9MAfx8D6tSJiUyuAwzLicaxlg==", + "dev": true, + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/jju": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/jju/-/jju-1.4.0.tgz", @@ -4287,6 +4355,18 @@ "get-func-name": "^2.0.0" } }, + "node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/magic-string": { "version": "0.29.0", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.29.0.tgz", @@ -4299,6 +4379,36 @@ "node": ">=12" } }, + "node_modules/make-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", + "dev": true, + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", + "dev": true, + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/md5-hex": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/md5-hex/-/md5-hex-3.0.1.tgz", @@ -5421,6 +5531,20 @@ "node": ">=6" } }, + "node_modules/test-exclude": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", + "integrity": "sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==", + "dev": true, + "dependencies": { + "@istanbuljs/schema": "^0.1.2", + "glob": "^7.1.4", + "minimatch": "^3.0.4" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -5703,6 +5827,20 @@ "requires-port": "^1.0.0" } }, + "node_modules/v8-to-istanbul": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.1.0.tgz", + "integrity": "sha512-6z3GW9x8G1gd+JIIgQQQxXuiJtCXeAjp6RaPEPLv62mH3iPHPxV6W3robxtCzNErRo6ZwTmzWhsbNvjyEBKzKA==", + "dev": true, + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.12", + "@types/istanbul-lib-coverage": "^2.0.1", + "convert-source-map": "^1.6.0" + }, + "engines": { + "node": ">=10.12.0" + } + }, "node_modules/validator": { "version": "13.9.0", "resolved": "https://registry.npmjs.org/validator/-/validator-13.9.0.tgz", @@ -6168,6 +6306,12 @@ "optional": true, "peer": true }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index 4d6acc8..ff71834 100644 --- a/package.json +++ b/package.json @@ -29,8 +29,9 @@ "homepage": "https://github.com/movie-web/providers#readme", "scripts": { "build": "vite build", - "test": "vitest run", + "test": "vitest run --coverage", "test:watch": "vitest", + "test:coverage": "vitest run --coverage", "lint": "eslint --ext .ts,.js src/", "lint:fix": "eslint --fix --ext .ts,.js src/", "lint:report": "eslint --ext .ts,.js --output-file eslint_report.json --format json src/", @@ -41,6 +42,7 @@ "@types/crypto-js": "^4.1.1", "@typescript-eslint/eslint-plugin": "^5.60.0", "@typescript-eslint/parser": "^5.60.0", + "@vitest/coverage-v8": "^0.34.3", "eslint": "^8.30.0", "eslint-config-airbnb-base": "^15.0.0", "eslint-config-prettier": "^8.5.0", From 637b3b635a9905fc782cdbda1d52ef1ceb0f0641 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 6 Sep 2023 15:09:52 +0200 Subject: [PATCH 20/23] fetcher seralization --- README.md | 5 ----- src/fetchers/body.ts | 24 ++++++++++++++++++++++++ src/fetchers/standardFetch.ts | 9 ++++++++- src/fetchers/types.ts | 2 +- 4 files changed, 33 insertions(+), 7 deletions(-) create mode 100644 src/fetchers/body.ts diff --git a/README.md b/README.md index c93054f..048a911 100644 --- a/README.md +++ b/README.md @@ -19,14 +19,9 @@ Todos: - makeStandardFetcher() - do all parameters get passed to real fetch as expected? - does serialisation work as expected? (formdata + json + string) - - do baseUrl settings work? - does json responses get automatically parsed? - - makeFullUrl() - - do slashes get converted correctly? - running individual scrapers - finish fetchers: - - make baseUrl param work - - proper serialization (with content-type headers) for standard fetcher - automatically parse json - error logging for failed scrapers - add all real providers diff --git a/src/fetchers/body.ts b/src/fetchers/body.ts new file mode 100644 index 0000000..3356953 --- /dev/null +++ b/src/fetchers/body.ts @@ -0,0 +1,24 @@ +import FormData = require('form-data'); + +import { FetcherOptions } from '@/fetchers/types'; + +export interface SeralizedBody { + headers: Record; + body: FormData | URLSearchParams | string | undefined; +} + +export function serializeBody(body: FetcherOptions['body']): SeralizedBody { + if (body === undefined || typeof body === 'string' || body instanceof URLSearchParams || body instanceof FormData) + return { + headers: {}, + body, + }; + + // serialize as JSON + return { + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(body), + }; +} diff --git a/src/fetchers/standardFetch.ts b/src/fetchers/standardFetch.ts index a4000eb..f07b6ac 100644 --- a/src/fetchers/standardFetch.ts +++ b/src/fetchers/standardFetch.ts @@ -1,5 +1,6 @@ import fetch from 'node-fetch'; +import { serializeBody } from '@/fetchers/body'; import { makeFullUrl } from '@/fetchers/common'; import { Fetcher } from '@/fetchers/types'; @@ -7,9 +8,15 @@ export function makeStandardFetcher(f: typeof fetch): Fetcher { const normalFetch: Fetcher = (url, ops) => { const fullUrl = makeFullUrl(url, ops); + const seralizedBody = serializeBody(ops.body); + return f(fullUrl, { method: ops.method, - body: JSON.stringify(ops.body), // TODO content type headers + proper serialization + headers: { + ...seralizedBody.headers, + ...ops.headers, + }, + body: seralizedBody.body, }); }; diff --git a/src/fetchers/types.ts b/src/fetchers/types.ts index ce522bc..2d14748 100644 --- a/src/fetchers/types.ts +++ b/src/fetchers/types.ts @@ -5,7 +5,7 @@ export type FetcherOptions = { headers?: Record; query?: Record; method?: 'GET' | 'POST'; - body?: Record | string | FormData; + body?: Record | string | FormData | URLSearchParams; }; export type DefaultedFetcherOptions = { From d325dab162d55bf527d2bfab38e1a5fea33b0f35 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 6 Sep 2023 17:04:43 +0200 Subject: [PATCH 21/23] error logging --- package.json | 2 +- src/main/events.ts | 1 + src/main/runner.ts | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index ff71834..48414ac 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "homepage": "https://github.com/movie-web/providers#readme", "scripts": { "build": "vite build", - "test": "vitest run --coverage", + "test": "vitest run", "test:watch": "vitest", "test:coverage": "vitest run --coverage", "lint": "eslint --ext .ts,.js src/", diff --git a/src/main/events.ts b/src/main/events.ts index 09f648d..5bd8e5d 100644 --- a/src/main/events.ts +++ b/src/main/events.ts @@ -3,6 +3,7 @@ export type UpdateEventStatus = 'success' | 'failure' | 'notfound' | 'pending'; export type UpdateEvent = { percentage: number; status: UpdateEventStatus; + error?: unknown; }; export type InitEvent = { diff --git a/src/main/runner.ts b/src/main/runner.ts index e09ea00..3a81bd2 100644 --- a/src/main/runner.ts +++ b/src/main/runner.ts @@ -1,8 +1,8 @@ import { UseableFetcher } from '@/fetchers/types'; import { FullScraperEvents } from '@/main/events'; import { ScrapeMedia } from '@/main/media'; -import { ProviderList } from '@/providers/all'; import { EmbedOutput, SourcererOutput } from '@/providers/base'; +import { ProviderList } from '@/providers/get'; import { Stream } from '@/providers/streams'; import { ScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; @@ -85,8 +85,8 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt ops.events?.update?.({ percentage: 100, status: 'failure', + error: err, }); - // TODO log error continue; } if (!output) throw new Error('Invalid media type'); @@ -139,8 +139,8 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt ops.events?.update?.({ percentage: 100, status: 'failure', + error: err, }); - // TODO log error continue; } From ee2b63034ef2587ddf2299697db58d4bb5edd681 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 6 Sep 2023 17:40:03 +0200 Subject: [PATCH 22/23] not found reasons + json parsing --- src/fetchers/standardFetch.ts | 9 ++++++--- src/main/events.ts | 3 ++- src/main/runner.ts | 2 ++ src/providers/sources/flixhq/index.ts | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/fetchers/standardFetch.ts b/src/fetchers/standardFetch.ts index f07b6ac..5110d0e 100644 --- a/src/fetchers/standardFetch.ts +++ b/src/fetchers/standardFetch.ts @@ -5,12 +5,11 @@ import { makeFullUrl } from '@/fetchers/common'; import { Fetcher } from '@/fetchers/types'; export function makeStandardFetcher(f: typeof fetch): Fetcher { - const normalFetch: Fetcher = (url, ops) => { + const normalFetch: Fetcher = async (url, ops) => { const fullUrl = makeFullUrl(url, ops); - const seralizedBody = serializeBody(ops.body); - return f(fullUrl, { + const res = await f(fullUrl, { method: ops.method, headers: { ...seralizedBody.headers, @@ -18,6 +17,10 @@ export function makeStandardFetcher(f: typeof fetch): Fetcher { }, body: seralizedBody.body, }); + + const isJson = res.headers.get('content-type')?.includes('application/json'); + if (isJson) return res.json(); + return res.text(); }; return normalFetch; diff --git a/src/main/events.ts b/src/main/events.ts index 5bd8e5d..485c86f 100644 --- a/src/main/events.ts +++ b/src/main/events.ts @@ -3,7 +3,8 @@ export type UpdateEventStatus = 'success' | 'failure' | 'notfound' | 'pending'; export type UpdateEvent = { percentage: number; status: UpdateEventStatus; - error?: unknown; + error?: unknown; // set when status is failure + reason?: string; // set when status is not-found }; export type InitEvent = { diff --git a/src/main/runner.ts b/src/main/runner.ts index 3a81bd2..09a6d2f 100644 --- a/src/main/runner.ts +++ b/src/main/runner.ts @@ -79,6 +79,7 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt ops.events?.update?.({ percentage: 100, status: 'notfound', + reason: err.message, }); continue; } @@ -133,6 +134,7 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt ops.events?.update?.({ percentage: 100, status: 'notfound', + reason: err.message, }); continue; } diff --git a/src/providers/sources/flixhq/index.ts b/src/providers/sources/flixhq/index.ts index f847502..c8559bd 100644 --- a/src/providers/sources/flixhq/index.ts +++ b/src/providers/sources/flixhq/index.ts @@ -11,7 +11,7 @@ export const flixhqScraper = makeSourcerer({ rank: 100, async scrapeMovie(ctx) { const id = await getFlixhqId(ctx, ctx.media); - if (!id) throw new NotFoundError(); + if (!id) throw new NotFoundError('no search results match'); const sources = await getFlixhqSources(ctx, id); const upcloudStream = sources.find((v) => v.embed.toLowerCase() === 'upcloud'); From b387e5ac1972f710639d80b04fa5cd569182838f Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 6 Sep 2023 17:58:28 +0200 Subject: [PATCH 23/23] bump version --- README.md | 5 +---- package.json | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 048a911..114051c 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ features: - scrape popular streaming websites - works in both browser and server-side -> This package is still WIP +> **This package is still WIP** Todos: - add tests @@ -21,9 +21,6 @@ Todos: - does serialisation work as expected? (formdata + json + string) - does json responses get automatically parsed? - running individual scrapers - - finish fetchers: - - automatically parse json - - error logging for failed scrapers - add all real providers Future todos: diff --git a/package.json b/package.json index 48414ac..65e000d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@movie-web/providers", - "version": "0.0.2", + "version": "0.0.3", "description": "Package that contains all the providers of movie-web", "main": "./lib/providers.umd.js", "types": "./lib/providers.d.ts",