Add metadata, embed scrape, source scrape and CORS config

src/index.ts

@@ -1,20 +1,44 @@
 import { Hono } from 'hono';
 import { streamSSE } from 'hono/streaming';
+import { cors } from 'hono/cors';
 import {
   ScrapeMedia,
   makeProviders,
   makeStandardFetcher,
   targets,
 } from '@movie-web/providers';
-import { ZodError } from 'zod';
-import { mediaSchema } from '@/schema';
+import { ZodError, z } from 'zod';
+import { embedSchema, scrapeAllSchema, sourceSchema } from '@/schema';
 import { validateTurnstile } from '@/turnstile';
 
 // hono doesn't export this type, so we retrieve it from a function
 type SSEStreamingApi = Parameters<Parameters<typeof streamSSE>['1']>['0'];
 
+const fetcher = makeStandardFetcher(fetch);
+
+const providers = makeProviders({
+  fetcher,
+  target: targets.BROWSER,
+});
+
 const app = new Hono();
 
+app.use('*', (context, next) => {
+  const allowedCorsHosts = ((context.env?.CORS_ALLOWED as string) ?? '').split(
+    ',',
+  );
+
+  return cors({
+    origin: (origin) => {
+      const hostname = new URL(origin).hostname;
+      if (allowedCorsHosts.includes(hostname)) {
+        return origin;
+      }
+      return '';
+    },
+  })(context, next);
+});
+
 let eventId = 0;
 async function writeSSEEvent(
   stream: SSEStreamingApi,
@@ -48,7 +72,7 @@ app.get('/scrape', async (context) => {
 
   let media: ScrapeMedia;
   try {
-    media = mediaSchema.parse(queryParams);
+    media = scrapeAllSchema.parse(queryParams);
   } catch (e) {
     if (e instanceof ZodError) {
       context.status(400);
@@ -58,13 +82,6 @@ app.get('/scrape', async (context) => {
     return context.text('An error has occurred!');
   }
 
-  const fetcher = makeStandardFetcher(fetch);
-
-  const providers = makeProviders({
-    fetcher,
-    target: targets.NATIVE,
-  });
-
   return streamSSE(context, async (stream) => {
     const output = await providers.runAll({
       media,
@@ -92,4 +109,106 @@ app.get('/scrape', async (context) => {
   });
 });
 
+app.get('/scrape/embed', async (context) => {
+  const queryParams = context.req.query();
+
+  const turnstileEnabled = Boolean(context.env?.TURNSTILE_ENABLED);
+
+  if (turnstileEnabled) {
+    const turnstileResponse = await validateTurnstile(context);
+
+    if (!turnstileResponse.success) {
+      context.status(401);
+      return context.text(
+        `Turnstile invalid, error codes: ${turnstileResponse.errorCodes.join(
+          ', ',
+        )}`,
+      );
+    }
+  }
+
+  let embedInput: z.infer<typeof embedSchema>;
+  try {
+    embedInput = embedSchema.parse(queryParams);
+  } catch (e) {
+    if (e instanceof ZodError) {
+      context.status(400);
+      return context.json(e.format());
+    }
+    context.status(500);
+    return context.text('An error has occurred!');
+  }
+
+  return streamSSE(context, async (stream) => {
+    const output = await providers.runEmbedScraper({
+      id: embedInput.id,
+      url: embedInput.url,
+      events: {
+        update(evt) {
+          writeSSEEvent(stream, 'update', evt);
+        },
+      },
+    });
+
+    if (output) {
+      return await writeSSEEvent(stream, 'completed', output);
+    }
+
+    return await writeSSEEvent(stream, 'noOutput', '');
+  });
+});
+
+app.get('/scrape/embed', async (context) => {
+  const queryParams = context.req.query();
+
+  const turnstileEnabled = Boolean(context.env?.TURNSTILE_ENABLED);
+
+  if (turnstileEnabled) {
+    const turnstileResponse = await validateTurnstile(context);
+
+    if (!turnstileResponse.success) {
+      context.status(401);
+      return context.text(
+        `Turnstile invalid, error codes: ${turnstileResponse.errorCodes.join(
+          ', ',
+        )}`,
+      );
+    }
+  }
+
+  let sourceInput: z.infer<typeof sourceSchema>;
+  try {
+    sourceInput = sourceSchema.parse(queryParams);
+  } catch (e) {
+    if (e instanceof ZodError) {
+      context.status(400);
+      return context.json(e.format());
+    }
+    context.status(500);
+    return context.text('An error has occurred!');
+  }
+
+  return streamSSE(context, async (stream) => {
+    const output = await providers.runSourceScraper({
+      id: sourceInput.id,
+      media: sourceInput,
+      events: {
+        update(evt) {
+          writeSSEEvent(stream, 'update', evt);
+        },
+      },
+    });
+
+    if (output) {
+      return await writeSSEEvent(stream, 'completed', output);
+    }
+
+    return await writeSSEEvent(stream, 'noOutput', '');
+  });
+});
+
+app.get('/metadata', async (context) => {
+  return context.json([providers.listEmbeds(), providers.listSources()]);
+});
+
 export default app;

src/schema.ts

@@ -2,7 +2,7 @@ import { z } from 'zod';
 
 export const tmdbIdSchema = z.string().regex(/^\d+$/);
 
-export const mediaSchema = z
+export const scrapeAllSchema = z
   .discriminatedUnion('type', [
     z.object({
       type: z.literal('movie'),
@@ -39,3 +39,14 @@ export const mediaSchema = z
       },
     };
   });
+
+export const embedSchema = z.object({
+  id: z.string(),
+  url: z.string(),
+});
+
+export const sourceSchema = scrapeAllSchema.and(
+  z.object({
+    id: z.string(),
+  }),
+);

wrangler.toml

@@ -4,5 +4,6 @@ workers_dev = true
 compatibility_date = "2023-12-17"
 
 [vars]
-TURNSTILE_ENABLED = true
+TURNSTILE_ENABLED = false
 TURNSTILE_SECRET = "1x0000000000000000000000000000000AA"
+CORS_ALLOWED = "localhost,movie-web.app,dev.movie-web.app"