1 Commits

Author SHA1 Message Date
William Oldham
fe04267494 Merge pull request #152 from movie-web/dev
Version 2.3.0: Disabling Showbox
2024-04-13 18:52:55 +01:00
77 changed files with 7945 additions and 3289 deletions

View File

@@ -1,7 +0,0 @@
root = true
[*]
end_of_line = lf
insert_final_newline = true
indent_size = 2
indent_style = space

View File

@@ -1,6 +1,4 @@
dist
node_modules
.output
public
# Ignore index due to prettier removing setext headers
*.index.md
.nuxt

View File

@@ -1,6 +1,8 @@
module.exports = {
extends: ['next', 'plugin:prettier/recommended'],
root: true,
extends: '@nuxt/eslint-config',
rules: {
'@next/next/no-img-element': 'off',
},
};
'vue/max-attributes-per-line': 'off',
'vue/multi-word-component-names': 'off'
}
}

View File

@@ -1 +0,0 @@
* text=auto eol=lf

View File

@@ -1 +0,0 @@
* @movie-web/project-leads

View File

@@ -1 +0,0 @@
Please visit the [main document at primary repository](https://github.com/movie-web/movie-web/blob/dev/.github/CODE_OF_CONDUCT.md).

View File

@@ -1 +0,0 @@
Please visit the [main document at primary repository](https://github.com/movie-web/movie-web/blob/dev/.github/CONTRIBUTING.md).

View File

@@ -1,49 +0,0 @@
name: "docs-deploy"
on:
push:
branches:
- master
jobs:
build:
name: Build
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- uses: pnpm/action-setup@v2
with:
version: 8
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: "20"
cache: pnpm
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Build
run: pnpm build
- name: Upload
uses: actions/upload-pages-artifact@v3
with:
path: ./out
deploy:
needs: build
permissions:
pages: write
id-token: write
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
runs-on: ubuntu-latest
steps:
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v4

View File

@@ -1,32 +0,0 @@
name: Linting and Testing
on:
push:
branches:
- master
pull_request:
jobs:
linting:
name: Run Linters
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- uses: pnpm/action-setup@v3
with:
version: 8
- name: Install Node.js
uses: actions/setup-node@v4
with:
node-version: 20
cache: 'pnpm'
- name: Install pnpm packages
run: pnpm install
- name: Run ESLint
run: pnpm run lint

5
.docs/.gitignore vendored
View File

@@ -2,11 +2,12 @@ node_modules
*.iml
.idea
*.log*
.nuxt
.vscode
.DS_Store
coverage
dist
sw.*
.env
out
.next
.output
.nuxt

View File

@@ -1,3 +0,0 @@
# Ignore index due to prettier removing setext headers
*.index.md
.github/CODEOWNERS

View File

@@ -1,4 +0,0 @@
{
"trailingComma": "all",
"singleQuote": true
}

View File

@@ -1,21 +0,0 @@
MIT License
Copyright (c) 2023 movie-web
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

17
.docs/app.config.ts Normal file
View File

@@ -0,0 +1,17 @@
export default defineAppConfig({
docus: {
title: '@movie-web/providers',
description: 'For all your media scraping needs',
socials: {
github: 'movie-web/providers',
},
image: '',
aside: {
level: 0,
exclude: [],
},
header: {
logo: false,
},
},
});

View File

@@ -0,0 +1,3 @@
code > span {
white-space: pre;
}

Binary file not shown.

View File

@@ -1,18 +0,0 @@
.logo {
border-radius: 5px;
margin-left: -0.5rem;
padding: 0.5rem;
transition: transform 100ms ease-in-out, background-color 100ms ease-in-out;
}
.logo > img {
height: 1.5rem;
}
.logo:hover {
background-color: rgba(var(--colors-bgLightest));
}
.logo:active {
transform: scale(1.05);
}

View File

@@ -1,11 +0,0 @@
import Link from 'next/link';
import classes from './Logo.module.css';
import logoUrl from '../public/icon-light.png';
export function Logo() {
return (
<Link href="/" className={classes.logo}>
<img src={logoUrl.src} alt="Logo of movie-web" />
</Link>
);
}

51
.docs/content/0.index.md Normal file
View File

@@ -0,0 +1,51 @@
---
title: "@movie-web/providers | For all your media scraping needs"
navigation: false
layout: page
---
::block-hero
---
cta:
- Get Started
- /get-started/introduction
secondary:
- Open on GitHub →
- https://github.com/movie-web/providers
snippet: npm i @movie-web/providers
---
#title
@movie-web/providers
#description
Easily scrape all sorts of media sites for content
::
::card-grid
#title
What's included
#root
:ellipsis
#default
::card{icon="vscode-icons:file-type-light-json"}
#title
Scrape popular streaming websites.
#description
Don't settle for just one media site for you content, use everything that's available.
::
::card{icon="codicon:source-control"}
#title
Multi-platform.
#description
Scrape from browser or server, whichever you prefer.
::
::card{icon="logos:typescript-icon-round"}
#title
Easy to use.
#description
Get started with scraping your favourite media sites with just 5 lines of code. Fully typed of course.
::
::

View File

@@ -1,8 +1,6 @@
---
title: 'Introduction'
---
# Introduction
## What is `@movie-web/providers` ?
## What is `@movie-web/providers`?
`@movie-web/providers` is the soul of [movie-web](https://github.com/movie-web/movie-web). It's a collection of scrapers of various streaming sites. It extracts the raw streams from those sites, so you can watch them without any extra fluff from the original sites.
@@ -13,4 +11,4 @@ We support many different environments, here are a few examples:
- In a native app, scrape in the app itself
- In a backend server, scrape on the server and give the streams to the client to watch.
To find out how to configure the library for your environment, You can read [How to use on X](/essentials/usage-on-x).
To find out how to configure the library for your environment, You can read [How to use on X](../2.essentials/0.usage-on-x.md).

View File

@@ -1,24 +1,30 @@
---
title: 'Quick Start'
---
# Quick start
## Installation
Let's get started with `@movie-web/providers`. First lets install the package.
```sh npm2yarn
::code-group
```bash [NPM]
npm install @movie-web/providers
```
```bash [Yarn]
yarn add @movie-web/providers
```
```bash [PNPM]
pnpm install @movie-web/providers
```
::
## Scrape your first item
To get started with scraping on the **server**, first you have to make an instance of the providers.
<Important>
This snippet will only work on a **server**. For other environments, check out [Usage on X](/essentials/usage-on-x).
</Important>
::alert{type="warning"}
This snippet will only work on a **server**. For other environments, check out [Usage on X](../2.essentials/0.usage-on-x.md).
::
```ts title="index.ts (server)" showLineNumbers
```ts [index.ts (server)]
import { makeProviders, makeStandardFetcher, targets } from '@movie-web/providers';
// this is how the library will make http requests
@@ -36,36 +42,19 @@ const providers = makeProviders({
Perfect. You now have an instance of the providers you can reuse everywhere.
Now let's scrape an item:
```ts title="index.ts (server)" showLineNumbers
import { ScrapeMedia, makeProviders, makeStandardFetcher, targets } from '@movie-web/providers';
const myFetcher = makeStandardFetcher(fetch);
const providers = makeProviders({
fetcher: myFetcher,
target: targets.NATIVE
});
const media: ScrapeMedia = {
```ts [index.ts (server)]
// fetch some data from TMDB
const media = {
type: 'movie',
title: "Oppenheimer",
releaseYear: 2023,
tmdbId: "872585"
};
async function fetchData() {
try {
const output = await providers.runAll({
media: media,
});
console.log("Output:",output)
} catch (error) {
console.error('Error occurred:', error);
}
title: "Hamilton",
releaseYear: 2020,
tmdbId: "556574"
}
fetchData();
const output = await providers.runAll({
media: media
})
```
Now we have our stream in the output variable. (If the output is `null` then nothing could be found.)
To find out how to use the streams, check out [Using streams](/essentials/using-streams).
To find out how to use the streams, check out [Using streams](../2.essentials/4.using-streams.md).

View File

@@ -0,0 +1,5 @@
# Examples
::alert{type="warning"}
There are no examples yet, stay tuned!
::

View File

@@ -102,9 +102,9 @@ title: 'Changelog'
# Version 2.0.0
<Warning>
::alert{type="warning"}
There are breaking changes in this list, make sure to read them thoroughly if you plan on updating.
</Warning>
::
**Development tooling:**
- Added integration test for browser. To make sure the package keeps working in the browser

View File

@@ -0,0 +1,2 @@
icon: ph:shooting-star-fill
navigation.redirect: /get-started/introduction

View File

@@ -24,7 +24,7 @@ const providers = makeProviders({
## Browser client-side
Using the provider package client-side requires a hosted version of simple-proxy.
Read more [about proxy fetchers](/essentials/fetchers#using-fetchers-on-the-browser).
Read more [about proxy fetchers](./2.fetchers.md#using-fetchers-on-the-browser).
```ts
import { makeProviders, makeStandardFetcher, targets } from '@movie-web/providers';
@@ -41,32 +41,27 @@ const providers = makeProviders({
## React native
To use the library in a react native app, you would also need a couple of polyfills to polyfill crypto and base64.
<Steps>
<Steps.Step>
### First install the polyfills:
```sh npm2yarn
npm install @react-native-anywhere/polyfill-base64 react-native-quick-crypto
```
</Steps.Step>
<Steps.Step>
### Add the polyfills to your app:
```ts
// Import in your entry file
import '@react-native-anywhere/polyfill-base64';
```
1. First install the polyfills:
```bash
npm install @react-native-anywhere/polyfill-base64 react-native-quick-crypto
```
And follow the [react-native-quick-crypto documentation](https://github.com/margelo/react-native-quick-crypto) to set up the crypto polyfill.
</Steps.Step>
<Steps.Step>
### Then you can use the library like this:
```ts
import { makeProviders, makeStandardFetcher, targets } from '@movie-web/providers';
2. Add the polyfills to your app:
```ts
// Import in your entry file
import '@react-native-anywhere/polyfill-base64';
```
const providers = makeProviders({
fetcher: makeStandardFetcher(fetch),
target: target.NATIVE,
consistentIpForRequests: true,
})
```
</Steps.Step>
</Steps>
And follow the [react-native-quick-crypto documentation](https://github.com/margelo/react-native-quick-crypto) to set up the crypto polyfill.
3. Then you can use the library like this:
```ts
import { makeProviders, makeStandardFetcher, targets } from '@movie-web/providers';
const providers = makeProviders({
fetcher: makeStandardFetcher(fetch),
target: target.NATIVE,
consistentIpForRequests: true,
})
```

View File

@@ -2,10 +2,10 @@
When creating provider controls, you will immediately be required to choose a target.
<Warning>
::alert{type="warning"}
A target is the device on which the stream will be played.
**Where the scraping is being run has nothing to do with the target**, only where the stream is finally played in the end is significant in choosing a target.
</Warning>
**Where the scraping is run has nothing to do with the target**, only where the stream is finally played in the end is significant in choosing a target.
::
#### Possible targets
- **`targets.BROWSER`** Stream will be played in a browser with CORS

View File

@@ -4,13 +4,13 @@ Streams can sometimes be quite picky on how they can be used. So here is a guide
## Essentials
All streams have the same common parameters :
- `Stream.type` : The type of stream. Either `hls` or `file`
- `Stream.id` : The id of this stream, unique per scraper output.
- `Stream.flags` : A list of flags that apply to this stream. Most people won't need to use it.
- `Stream.captions` : A list of captions/subtitles for this stream.
- `Stream.headers` : Either undefined or a key value object of headers you must set to use the stream.
- `Stream.preferredHeaders` : Either undefined or a key value object of headers you may want to set if you want optimal playback - but not required.
All streams have the same common parameters:
- `Stream.type`: The type of stream. Either `hls` or `file`
- `Stream.id`: The id of this stream, unique per scraper output.
- `Stream.flags`: A list of flags that apply to this stream. Most people won't need to use it.
- `Stream.captions`: A list of captions/subtitles for this stream.
- `Stream.headers`: Either undefined or a key value object of headers you must set to use the stream.
- `Stream.preferredHeaders`: Either undefined or a key value object of headers you may want to set if you want optimal playback - but not required.
Now let's delve deeper into how to watch these streams!
@@ -22,7 +22,7 @@ These streams have an extra property `Stream.playlist` which contains the m3u8 p
Here is a code sample of how to use HLS streams in web context using hls.js
```html
<script src="https ://cdn.jsdelivr.net/npm/hls.js@1"></script>
<script src="https://cdn.jsdelivr.net/npm/hls.js@1"></script>
<video id="video"></video>
<script>
@@ -39,17 +39,17 @@ Here is a code sample of how to use HLS streams in web context using hls.js
## Streams with type `file`
File streams are quite easy to use, they just return a new property : `Stream.qualities`.
File streams are quite easy to use, they just return a new property: `Stream.qualities`.
This property is a map of quality and a stream file. So if you want to get 1080p quality you do `stream["1080"]` to get your stream file. It will return undefined if that quality is absent.
The possibly qualities are : `unknown`, `360`, `480`, `720`, `1080`, `4k`.
The possibly qualities are: `unknown`, `360`, `480`, `720`, `1080`, `4k`.
File based streams are always guaranteed to have one quality.
Once you get a streamfile, you have the following parameters :
- `StreamFile.type` : Right now it can only be `mp4`.
- `StreamFile.url` : The URL linking to the video file.
Once you get a streamfile, you have the following parameters:
- `StreamFile.type`: Right now it can only be `mp4`.
- `StreamFile.url`: The URL linking to the video file.
Here is a code sample of how to watch a file based stream in a browser :
Here is a code sample of how to watch a file based stream in a browser:
```html
<video id="video"></video>
@@ -72,13 +72,13 @@ If your target is set to `BROWSER`, headers will never be required, as it's not
## Using captions/subtitles
All streams have a list of captions at `Stream.captions`. The structure looks like this :
All streams have a list of captions at `Stream.captions`. The structure looks like this:
```ts
type Caption = {
type : CaptionType; // Language type, either "srt" or "vtt"
id : string; // Unique per stream
url : string; // The URL pointing to the subtitle file
hasCorsRestrictions : boolean; // If true, you will need to proxy it if you're running in a browser
language : string; // Language code of the caption
type: CaptionType; // Language type, either "srt" or "vtt"
id: string; // Unique per stream
url: string; // The URL pointing to the subtitle file
hasCorsRestrictions: boolean; // If true, you will need to proxy it if you're running in a browser
language: string; // Language code of the caption
};
```

View File

@@ -0,0 +1,3 @@
icon: ph:info-fill
navigation.redirect: /essentials/usage
navigation.title: "Get started"

View File

@@ -1,11 +1,11 @@
# Sources vs embeds
<Warning>
::alert{type="warning"}
This page isn't quite done yet, stay tuned!
</Warning>
::
{/*
<!--
TODO
- How do sources and embeds differ
- How do sources and embeds interact
*/}
-->

View File

@@ -1,12 +1,12 @@
# New providers
<Warning>
::alert{type="warning"}
This page isn't quite done yet, stay tuned!
</Warning>
::
{/*
<!--
TODO
- How to make new sources or embeds
- Ranking
- Link to flags
*/}
-->

View File

@@ -6,5 +6,5 @@ For example, some sources only give back content that has the CORS headers set t
This concept is applied in multiple away across the library.
## Flag options
- `CORS_ALLOWED` : Headers from the output streams are set to allow any origin.
- `IP_LOCKED` : The streams are locked by IP: requester and watcher must be the same.
- `CORS_ALLOWED`: Headers from the output streams are set to allow any origin.
- `IP_LOCKED`: The streams are locked by IP: requester and watcher must be the same.

View File

@@ -0,0 +1,3 @@
icon: ph:atom-fill
navigation.redirect: /in-depth/sources-and-embeds
navigation.title: "In-depth"

View File

@@ -1,16 +1,16 @@
# Development / contributing
<Warning>
::alert{type="warning"}
This page isn't quite done yet, stay tuned!
</Warning>
::
{/*
<!--
TODO
- Development setup
- How to make new sources/embeds (link to the page)
- How to use the fetchers, when to use proxiedFetcher
- How to use the context
*/}
-->
## Testing using the CLI
@@ -27,7 +27,7 @@ Then make sure you've run `npm i` to get all the dependencies.
To run the CLI without needing to learn all the arguments, simply run the following command and go with the flow.
```sh npm2yarn
```sh
npm run cli
```
@@ -63,10 +63,10 @@ npm run cli -- -sid febbox-mp4 -u URL_HERE
### Fetcher options
The CLI comes with a few built-in fetchers:
- `node-fetch` : Fetch using the "node-fetch" library.
- `native` : Use the new fetch built into Node.JS (undici).
- `browser` : Start up headless chrome, and run the library in that context using a proxied fetcher.
- `node-fetch`: Fetch using the "node-fetch" library.
- `native`: Use the new fetch built into Node.JS (undici).
- `browser`: Start up headless chrome, and run the library in that context using a proxied fetcher.
<Warning>
::alert{type="warning"}
The browser fetcher will require you to run `npm run build` before running the CLI. Otherwise you will get outdated results.
</Warning>
::

View File

@@ -0,0 +1,3 @@
icon: ph:aperture-fill
navigation.redirect: /extra-topics/development
navigation.title: "Extra topics"

View File

@@ -6,10 +6,8 @@ You can attach events if you need to know what is going on while it is processin
## Example
```ts
import { ScrapeMedia, targets } from '@movie-web/providers';
// media from TMDB
const media : ScrapeMedia = {
const media = {
type: 'movie',
title: 'Hamilton',
releaseYear: 2020,

View File

@@ -5,10 +5,10 @@ Run a specific source scraper and get its emitted streams.
## Example
```ts
import { ScrapeMedia , SourcererOutput, NotFoundError } from '@movie-web/providers';
import { SourcererOutput, NotFoundError } from '@movie-web/providers';
// media from TMDB
const media : ScrapeMedia = {
const media = {
type: 'movie',
title: 'Hamilton',
releaseYear: 2020,

5
.docs/next-env.d.ts vendored
View File

@@ -1,5 +0,0 @@
/// <reference types="next" />
/// <reference types="next/image-types/global" />
// NOTE: This file should not be edited
// see https://nextjs.org/docs/basic-features/typescript for more information.

View File

@@ -1,10 +0,0 @@
import { guider } from '@neato/guider';
const withGuider = guider({
themeConfig: './theme.config.tsx',
});
export default withGuider({
output: 'export',
basePath: '/providers',
});

21
.docs/nuxt.config.ts Executable file
View File

@@ -0,0 +1,21 @@
export default defineNuxtConfig({
// https://github.com/nuxt-themes/docus
extends: '@nuxt-themes/docus',
css: [
'@/assets/css/main.css',
],
build: {
transpile: [
"chalk"
]
},
modules: [
// https://github.com/nuxt-modules/plausible
'@nuxtjs/plausible',
// https://github.com/nuxt/devtools
'@nuxt/devtools'
]
})

View File

@@ -1,28 +1,22 @@
{
"name": "providers-docs",
"version": "0.2.0",
"version": "0.1.0",
"private": true,
"scripts": {
"dev": "next dev",
"build": "next build",
"start": "next start",
"lint": "next lint",
"lint:fix": "next lint --fix"
"dev": "nuxi dev",
"build": "nuxi build",
"generate": "nuxi generate",
"preview": "nuxi preview",
"lint": "eslint .",
"preinstall": "npx -y only-allow pnpm"
},
"devDependencies": {
"@types/react": "18.2.73",
"eslint": "^8.56.0",
"eslint-config-next": "^14.1.4",
"eslint-config-prettier": "^9.1.0",
"eslint-plugin-prettier": "^5.1.2",
"prettier": "^3.1.1",
"typescript": "5.4.3"
},
"dependencies": {
"@neato/guider": "^0.1.5",
"next": "^14.1.4",
"next-seo": "^6.5.0",
"react": "^18.2.0",
"react-dom": "^18.2.0"
"@nuxt-themes/docus": "^1.13.1",
"@nuxt/devtools": "^1.0.1",
"@nuxt/eslint-config": "^0.1.1",
"@nuxtjs/plausible": "^0.2.1",
"@types/node": "^20.4.0",
"eslint": "^8.44.0",
"nuxt": "^3.6.2"
}
}

View File

@@ -1,3 +0,0 @@
import { createNotFoundPage } from '@neato/guider/client';
export default createNotFoundPage();

View File

@@ -1,4 +0,0 @@
import '@neato/guider/style.css';
import { createGuiderApp } from '@neato/guider/client';
export default createGuiderApp();

View File

@@ -1,3 +0,0 @@
import { createRedirect } from '@neato/guider/client';
export default createRedirect({ to: '/api-reference/makeProviders' });

View File

@@ -1,3 +0,0 @@
import { createRedirect } from '@neato/guider/client';
export default createRedirect({ to: '/essentials/usage-on-x' });

View File

@@ -1,3 +0,0 @@
import { createRedirect } from '@neato/guider/client';
export default createRedirect({ to: '/in-depth/sources-vs-embeds' });

View File

@@ -1,7 +0,0 @@
---
title: 'Examples'
---
<Note>
Coming soon
</Note>

View File

@@ -1,3 +0,0 @@
import { createRedirect } from '@neato/guider/client';
export default createRedirect({ to: '/get-started/introduction' });

View File

@@ -1,3 +0,0 @@
import { createRedirect } from '@neato/guider/client';
export default createRedirect({ to: '/in-depth/sources-vs-embeds' });

View File

@@ -1,42 +0,0 @@
import {
Button,
Card,
CardGrid,
GuiderLayout,
Hero,
} from '@neato/guider/client';
export default function LandingPage() {
return (
<GuiderLayout meta={{ layout: 'page' }}>
<Hero>
<Hero.Badge title="V2.3.0" to="/get-started/changelog">
See changelog for more
</Hero.Badge>
<Hero.Title>@movie-web/providers</Hero.Title>
<Hero.Subtitle>
Easily scrape all sorts of media sites for content.
</Hero.Subtitle>
<Hero.Actions>
<Button to="/get-started/introduction">Get Started</Button>
<Button to="https://github.com/movie-web/providers" type="secondary">
Open on GitHub
</Button>
</Hero.Actions>
</Hero>
<CardGrid>
<Card icon="mdi:code-json" title="Scrape popular streaming websites.">
Don&apos;t settle for just one media site for you content, use
everything that&apos;s available.
</Card>
<Card icon="mdi:git" title="Multi-platform.">
Scrape from browser or server, whichever you prefer.
</Card>
<Card icon="mdi:language-typescript" title="Easy to use.">
Get started with scraping your favourite media sites with just 5 lines
of code. Fully typed of course.
</Card>
</CardGrid>
</GuiderLayout>
);
}

10120
.docs/pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

Binary file not shown.

Before

Width:  |  Height:  |  Size: 887 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 328 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 28 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 27 KiB

8
.docs/renovate.json Executable file
View File

@@ -0,0 +1,8 @@
{
"extends": [
"@nuxtjs"
],
"lockFileMaintenance": {
"enabled": true
}
}

View File

@@ -1,112 +0,0 @@
import { defineTheme, directory, group, link, social } from '@neato/guider/theme';
import { Logo } from './components/Logo';
import { NextSeo } from 'next-seo';
import coverUrl from "./public/cover.png";
import faviconUrl from "./public/favicon.ico";
export default defineTheme({
github: "movie-web/providers",
contentFooter: {
text: "Made with 💜",
editRepositoryBase: "https://github.com/movie-web/providers",
socials: [
social.github("https://github.com/movie-web/providers"),
social.discord("https://movie-web.github.io/links/discord"),
]
},
meta: (pageMeta) => (
<NextSeo {...{
title: `${pageMeta.title ?? "For all your media scraping needs"} | movie-web`,
description: pageMeta.description ?? "movie-web/providers : Easily scrape all sorts of media sites for content.",
openGraph: {
images: [{
url: coverUrl.src,
}],
title: `${pageMeta.title ?? "For all your media scraping needs"} | movie-web`,
description: pageMeta.description ?? "movie-web/providers : Easily scrape all sorts of media sites for content.",
},
twitter: {
cardType: 'summary_large_image',
},
additionalLinkTags: [
{
href: faviconUrl.src,
rel: "icon",
type: "image/x-icon",
}
]
}} />
),
settings: {
logo: () => <Logo />,
colors: {
primary: "#E67070",
primaryLighter: "#E59595",
primaryDarker: "#D21818",
background: "#000000",
backgroundLighter: "#141414",
backgroundLightest: "#292929",
backgroundDarker: "#000000",
line: "#404040",
text: "#B3B3B3",
textLighter: "#CCCCCC",
textHighlight: "#cccccc",
codeWarning: '#222D20',
codeError: '#2B1B1F',
codeGreen: '#0B2823',
codeHighlight: '#0E2429',
codeWordHighlight: '#365C68',
semanticTip: '#39B864',
semanticTipLighter: '#75F2B6',
semanticNote: '#817EF3',
semanticNoteLighter: '#B9B8FC',
semanticImportant: '#A958E8',
semanticImportantLighter: '#D3A2F9',
semanticWarning: '#fff708',
semanticWarningLighter: '#faf8b1',
semanticCaution: '#FC6359',
semanticCautionLighter: '#FFA59F',
},
backgroundPattern: 'flare',
},
directories: [
directory("main", {
sidebar: [
group("Get Started", [
link("Introduction", "/get-started/introduction"),
link("Quickstart", "/get-started/quick-start"),
link("Examples", "/get-started/examples"),
link("Changelog", "/get-started/changelog"),
]),
group("Essentials", [
link("Usage on X", "/essentials/usage-on-x"),
link("Targets", "/essentials/targets"),
link("Fetchers", "/essentials/fetchers"),
link("Customize Providers", "/essentials/customize-providers"),
link("Using Streams", "/essentials/using-streams"),
]),
group("In Depth", [
link("Sources vs Embeds", "/in-depth/sources-vs-embeds"),
link("New Providers", "/in-depth/new-providers"),
link("Flags", "/in-depth/flags"),
]),
group("Extra Topics", [
link("Development and Contributing", "/extra-topics/development"),
]),
group("Api Reference", [
link("makeProviders", "/api-reference/makeProviders"),
link("ProviderControls.runAll", "/api-reference/ProviderControlsRunAll"),
link("ProviderControls.runSourceScraper", "/api-reference/ProviderControlsrunSourceScraper"),
link("ProviderControls.runEmbedScraper", "/api-reference/ProviderControlsrunEmbedScraper"),
link("ProviderControls.listSources", "/api-reference/ProviderControlslistSources"),
link("ProviderControls.listEmbeds", "/api-reference/ProviderControlslistEmbeds"),
link("ProviderControls.getMetadata", "/api-reference/ProviderControlsgetMetadata"),
link("makeStandardFetcher", "/api-reference/makeStandardFetcher"),
link("makeSimpleProxyFetcher", "/api-reference/makeSimpleProxyFetcher"),
])
]
})
],
});

18
.docs/tokens.config.ts Normal file
View File

@@ -0,0 +1,18 @@
import { defineTheme } from 'pinceau'
export default defineTheme({
color: {
primary: {
50: "#F5E5FF",
100: "#E7CCFF",
200: "#D4A9FF",
300: "#BE85FF",
400: "#A861FF",
500: "#8E3DFF",
600: "#7F36D4",
700: "#662CA6",
800: "#552578",
900: "#441E49"
}
}
})

View File

@@ -1,28 +1,3 @@
{
"compilerOptions": {
"lib": [
"dom",
"dom.iterable",
"esnext"
],
"allowJs": true,
"skipLibCheck": true,
"strict": false,
"noEmit": true,
"incremental": true,
"esModuleInterop": true,
"module": "esnext",
"moduleResolution": "Bundler",
"resolveJsonModule": true,
"isolatedModules": true,
"jsx": "preserve"
},
"include": [
"next-env.d.ts",
"**/*.ts",
"**/*.tsx"
],
"exclude": [
"node_modules"
]
"extends": "./.nuxt/tsconfig.json"
}

View File

@@ -19,14 +19,11 @@ import { kissAsianScraper } from '@/providers/sources/kissasian/index';
import { lookmovieScraper } from '@/providers/sources/lookmovie';
import { remotestreamScraper } from '@/providers/sources/remotestream';
import { showboxScraper } from '@/providers/sources/showbox/index';
import { tugaflixScraper } from '@/providers/sources/tugaflix';
import { vidsrcScraper } from '@/providers/sources/vidsrc/index';
import { zoechipScraper } from '@/providers/sources/zoechip';
import { bflixScraper } from './embeds/bflix';
import { closeLoadScraper } from './embeds/closeload';
import { fileMoonScraper } from './embeds/filemoon';
import { fileMoonMp4Scraper } from './embeds/filemoon/mp4';
import { ridooScraper } from './embeds/ridoo';
import { smashyStreamOScraper } from './embeds/smashystream/opstream';
import { smashyStreamFScraper } from './embeds/smashystream/video1';
@@ -41,7 +38,6 @@ import { wootlyScraper } from './embeds/wootly';
import { goojaraScraper } from './sources/goojara';
import { hdRezkaScraper } from './sources/hdrezka';
import { nepuScraper } from './sources/nepu';
import { nitesScraper } from './sources/nites';
import { primewireScraper } from './sources/primewire';
import { ridooMoviesScraper } from './sources/ridomovies';
import { smashyStreamScraper } from './sources/smashystream';
@@ -69,9 +65,7 @@ export function gatherAllSources(): Array<Sourcerer> {
primewireScraper,
warezcdnScraper,
insertunitScraper,
nitesScraper,
soaperTvScraper,
tugaflixScraper,
];
}
@@ -93,7 +87,6 @@ export function gatherAllEmbeds(): Array<Embed> {
ridooScraper,
closeLoadScraper,
fileMoonScraper,
fileMoonMp4Scraper,
vidplayScraper,
wootlyScraper,
doodScraper,
@@ -105,6 +98,5 @@ export function gatherAllEmbeds(): Array<Embed> {
vTubeScraper,
warezcdnembedHlsScraper,
warezcdnembedMp4Scraper,
bflixScraper,
];
}

View File

@@ -1,42 +0,0 @@
import { unpack } from 'unpacker';
import { makeEmbed } from '@/providers/base';
const evalCodeRegex = /eval\((.*)\)/g;
const mp4Regex = /https?:\/\/.*\.mp4/;
export const bflixScraper = makeEmbed({
id: 'bflix',
name: 'bFlix',
rank: 113,
scrape: async (ctx) => {
const mainPage = await ctx.proxiedFetcher<string>(ctx.url);
const evalCode = mainPage.match(evalCodeRegex);
if (!evalCode) throw new Error('Failed to find eval code');
const unpacked = unpack(evalCode[0]);
const file = unpacked.match(mp4Regex);
if (!file?.[0]) throw new Error('Failed to find file');
return {
stream: [
{
id: 'primary',
type: 'file',
flags: [],
captions: [],
qualities: {
unknown: {
type: 'mp4',
url: file[0],
},
},
headers: {
Referer: 'https://bflix.gs/',
},
},
],
};
},
});

View File

@@ -11,7 +11,7 @@ const fileRegex = /file:"(.*?)"/g;
export const fileMoonScraper = makeEmbed({
id: 'filemoon',
name: 'Filemoon',
rank: 300,
rank: 400,
scrape: async (ctx) => {
const embedRes = await ctx.proxiedFetcher<string>(ctx.url, {
headers: {

View File

@@ -1,37 +0,0 @@
import { NotFoundError } from '@/utils/errors';
import { makeEmbed } from '../../base';
import { fileMoonScraper } from './index';
export const fileMoonMp4Scraper = makeEmbed({
id: 'filemoon-mp4',
name: 'Filemoon MP4',
rank: 400,
scrape: async (ctx) => {
const result = await fileMoonScraper.scrape(ctx);
if (!result.stream) throw new NotFoundError('Failed to find result');
if (result.stream[0].type !== 'hls') throw new NotFoundError('Failed to find hls stream');
const url = result.stream[0].playlist.replace(/\/hls2\//, '/download/').replace(/\.m3u8/, '.mp4');
return {
stream: [
{
id: 'primary',
type: 'file',
qualities: {
unknown: {
type: 'mp4',
url,
},
},
flags: [],
captions: result.stream[0].captions,
},
],
};
},
});

View File

@@ -1,79 +0,0 @@
import { load } from 'cheerio';
import { SourcererOutput, makeSourcerer } from '@/providers/base';
import { compareMedia } from '@/utils/compare';
import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
import { NotFoundError } from '@/utils/errors';
const baseUrl = 'https://w1.nites.is';
async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promise<SourcererOutput> {
const searchPage = await ctx.proxiedFetcher('/wp-admin/admin-ajax.php', {
baseUrl,
method: 'POST',
body: new URLSearchParams({
action: 'ajax_pagination',
query_vars: 'mixed',
search: ctx.media.title,
}),
});
const $search = load(searchPage);
const searchResults: { title: string; year: number; url: string }[] = [];
$search('li').each((_, element) => {
const title = $search(element).find('.entry-title').first().text().trim();
const year = parseInt($search(element).find('.year').first().text().trim(), 10);
const url = $search(element).find('.lnk-blk').attr('href');
if (!title || !year || !url) return;
searchResults.push({ title, year, url });
});
let watchPageUrl = searchResults.find((x) => x && compareMedia(ctx.media, x.title, x.year))?.url;
if (!watchPageUrl) throw new NotFoundError('No watchable item found');
if (ctx.media.type === 'show') {
const match = watchPageUrl.match(/\/series\/([^/]+)\/?/);
if (!match) throw new Error('Failed to parse watch page url');
watchPageUrl = watchPageUrl.replace(
`/series/${match[1]}`,
`/episode/${match[1]}-${ctx.media.season.number}x${ctx.media.episode.number}`,
);
}
const watchPage = load(await ctx.proxiedFetcher(watchPageUrl));
// it embeds vidsrc when it bflix does not has the stream
// i think all shows embed vidsrc, not sure
const embedUrl = watchPage('ul.bx-lst li a:contains("- Bflix")')
.closest('aside')
.next('div.video-options')
.find('iframe')
.attr('data-lazy-src');
if (!embedUrl) throw new Error('Failed to find embed url');
const embedPage = load(await ctx.proxiedFetcher(embedUrl));
const url = embedPage('iframe').attr('src');
if (!url) throw new Error('Failed to find embed url');
return {
embeds: [
{
embedId: 'bflix',
url,
},
],
};
}
export const nitesScraper = makeSourcerer({
id: 'nites',
name: 'Nites',
rank: 90,
flags: [],
scrapeMovie: comboScraper,
scrapeShow: comboScraper,
});

View File

@@ -1,21 +0,0 @@
import { load } from 'cheerio';
export const baseUrl = 'https://tugaflix.best/';
export function parseSearch(page: string): { title: string; year?: number; url: string }[] {
const results: { title: string; year?: number; url: string }[] = [];
const $ = load(page);
$('.items .poster').each((_, element) => {
const $link = $(element).find('a');
const url = $link.attr('href');
// ex title: Home Alone (1990)
const [, title, year] = $link.attr('title')?.match(/^(.*?)\s*(?:\((\d{4})\))?\s*$/) || [];
if (!title || !url) return;
// tiles dont always have the year
results.push({ title, year: year ? parseInt(year, 10) : undefined, url });
});
return results;
}

View File

@@ -1,116 +0,0 @@
import { load } from 'cheerio';
import { flags } from '@/entrypoint/utils/targets';
import { SourcererEmbed, makeSourcerer } from '@/providers/base';
import { compareMedia } from '@/utils/compare';
import { NotFoundError } from '@/utils/errors';
import { baseUrl, parseSearch } from './common';
export const tugaflixScraper = makeSourcerer({
id: 'tugaflix',
name: 'Tugaflix',
rank: 73,
flags: [flags.IP_LOCKED],
scrapeMovie: async (ctx) => {
const searchResults = parseSearch(
await ctx.proxiedFetcher<string>('/filmes/', {
baseUrl,
query: {
s: ctx.media.title,
},
}),
);
if (searchResults.length === 0) throw new NotFoundError('No watchable item found');
const url = searchResults.find((x) => x && compareMedia(ctx.media, x.title, x.year))?.url;
if (!url) throw new NotFoundError('No watchable item found');
const videoPage = await ctx.proxiedFetcher<string>(url, {
method: 'POST',
body: new URLSearchParams({ play: '' }),
});
const $ = load(videoPage);
const embeds: SourcererEmbed[] = [];
for (const element of $('.play a')) {
const embedUrl = $(element).attr('href');
if (!embedUrl) continue;
const embedPage = await ctx.proxiedFetcher.full(
embedUrl.startsWith('https://') ? embedUrl : `https://${embedUrl}`,
);
const finalUrl = load(embedPage.body)('a:contains("Download Filme")').attr('href');
if (!finalUrl) continue;
if (finalUrl.includes('streamtape')) {
embeds.push({
embedId: 'streamtape',
url: finalUrl,
});
// found doodstream on a few shows, maybe movies use it too?
// the player 2 is just streamtape in a custom player
} else if (finalUrl.includes('dood')) {
embeds.push({
embedId: 'dood',
url: finalUrl,
});
}
}
return {
embeds,
};
},
scrapeShow: async (ctx) => {
const searchResults = parseSearch(
await ctx.proxiedFetcher<string>('/series/', {
baseUrl,
query: {
s: ctx.media.title,
},
}),
);
if (searchResults.length === 0) throw new NotFoundError('No watchable item found');
const url = searchResults.find((x) => x && compareMedia(ctx.media, x.title, x.year))?.url;
if (!url) throw new NotFoundError('No watchable item found');
const s = ctx.media.season.number < 10 ? `0${ctx.media.season.number}` : ctx.media.season.number.toString();
const e = ctx.media.episode.number < 10 ? `0${ctx.media.episode.number}` : ctx.media.episode.number.toString();
const videoPage = await ctx.proxiedFetcher(url, {
method: 'POST',
body: new URLSearchParams({ [`S${s}E${e}`]: '' }),
});
const embedUrl = load(videoPage)('iframe[name="player"]').attr('src');
if (!embedUrl) throw new Error('Failed to find iframe');
const playerPage = await ctx.proxiedFetcher(embedUrl.startsWith('https:') ? embedUrl : `https:${embedUrl}`, {
method: 'POST',
body: new URLSearchParams({ submit: '' }),
});
const embeds: SourcererEmbed[] = [];
const finalUrl = load(playerPage)('a:contains("Download Episodio")').attr('href');
if (finalUrl?.includes('streamtape')) {
embeds.push({
embedId: 'streamtape',
url: finalUrl,
});
} else if (finalUrl?.includes('dood')) {
embeds.push({
embedId: 'dood',
url: finalUrl,
});
}
return {
embeds,
};
},
});

View File

@@ -60,16 +60,10 @@ const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Pr
const urlWithSubtitles = embedArr.find((v) => v.source === 'Vidplay' && v.url.includes('sub.info'))?.url;
const subtitleUrl = urlWithSubtitles ? new URL(urlWithSubtitles).searchParams.get('sub.info') : null;
if (subtitleUrl) fullUrl.searchParams.set('sub.info', subtitleUrl);
embeds.push(
{
embedId: 'filemoon',
url: fullUrl.toString(),
},
{
embedId: 'filemoon-mp4',
url: fullUrl.toString(),
},
);
embeds.push({
embedId: 'filemoon',
url: fullUrl.toString(),
});
}
}