Merge pull request #13 from movie-web/dev

Version 1.1: Progress and Bookmark Bulk Importing

package.json

@@ -1,6 +1,6 @@
 {
   "name": "backend",
-  "version": "1.0.1",
+  "version": "1.1.0",
   "private": true,
   "homepage": "https://github.com/movie-web/backend",
   "engines": {
@@ -15,7 +15,9 @@
     "build:pre": "rimraf dist/",
     "build:compile": "tsc && tsc-alias",
     "preinstall": "npx -y only-allow pnpm",
-    "migration:create": "npx -y mikro-orm migration:create"
+    "migration:create": "npx -y mikro-orm migration:create",
+    "migration:up": "npx -y mikro-orm migration:up",
+    "migration:down": "npx -y mikro-orm migration:down"
   },
   "mikro-orm": {
     "useTsNode": true,

src/config/fragments/dev.ts

@@ -2,7 +2,7 @@ import { FragmentSchema } from '@/config/fragments/types';
 
 export const devFragment: FragmentSchema = {
   server: {
-    cors: 'http://localhost:5173',
+    allowAnySite: true,
     trustProxy: true,
   },
   logging: {

src/config/index.ts

@@ -8,7 +8,7 @@ const fragments = {
   dockerdev: dockerFragment,
 };
 
-export const version = '1.0.0';
+export const version = process.env.npm_package_version ?? 'unknown';
 
 export const conf = createConfigLoader()
   .addFromEnvironment('MWB_')

src/config/orm.ts

@@ -0,0 +1,20 @@
+import { createConfigLoader } from 'neat-config';
+import { z } from 'zod';
+
+export const ormConfigSchema = z.object({
+  postgres: z.object({
+    // connection URL for postgres database
+    connection: z.string(),
+  }),
+});
+
+export const ormConf = createConfigLoader()
+  .addFromEnvironment('MWB_')
+  .addFromCLI('mwb-')
+  .addFromFile('.env', {
+    prefix: 'MWB_',
+  })
+  .addFromFile('config.json')
+  .addZodSchema(ormConfigSchema)
+  .freeze()
+  .load();

src/config/schema.ts

@@ -9,6 +9,10 @@ export const configSchema = z.object({
       // space seperated list of allowed cors domains
       cors: z.string().default(''),
 
+      // disable cross origin restrictions, allow any site.
+      // overwrites the cors option above
+      allowAnySite: z.coerce.boolean().default(false),
+
       // should it trust reverse proxy headers? (for ip gathering)
       trustProxy: z.coerce.boolean().default(false),
 
@@ -37,6 +41,10 @@ export const configSchema = z.object({
     // will always keep the database schema in sync with the connected database
     // it is extremely destructive, do not use it EVER in production
     syncSchema: z.coerce.boolean().default(false),
+
+    // Enable debug logging for MikroORM - Outputs queries and entity management logs
+    // Do NOT use in production, leaks all sensitive data
+    debugLogging: z.coerce.boolean().default(false),
   }),
   crypto: z.object({
     // session secret. used for signing session tokens

src/db/migrations/.snapshot-movie_web.json

@@ -184,6 +184,24 @@
           "nullable": true,
           "mappedType": "string"
         },
+        "season_number": {
+          "name": "season_number",
+          "type": "int",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": true,
+          "mappedType": "integer"
+        },
+        "episode_number": {
+          "name": "episode_number",
+          "type": "int",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": true,
+          "mappedType": "integer"
+        },
         "meta": {
           "name": "meta",
           "type": "jsonb",
@@ -345,20 +363,29 @@
         },
         "error_message": {
           "name": "error_message",
-          "type": "varchar(255)",
+          "type": "text",
           "unsigned": false,
           "autoincrement": false,
           "primary": false,
           "nullable": true,
-          "mappedType": "string"
+          "mappedType": "text"
         },
         "full_error": {
           "name": "full_error",
-          "type": "varchar(255)",
+          "type": "text",
           "unsigned": false,
           "autoincrement": false,
           "primary": false,
           "nullable": true,
+          "mappedType": "text"
+        },
+        "hostname": {
+          "name": "hostname",
+          "type": "varchar(255)",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": false,
           "mappedType": "string"
         }
       },

src/db/migrations/Migration20231105150807.ts

@@ -0,0 +1,14 @@
+import { Migration } from '@mikro-orm/migrations';
+
+export class Migration20231105150807 extends Migration {
+  async up(): Promise<void> {
+    this.addSql(
+      'alter table "progress_items" add column "season_number" int null, add column "episode_number" int null;',
+    );
+  }
+
+  async down(): Promise<void> {
+    this.addSql('alter table "progress_items" drop column "season_number";');
+    this.addSql('alter table "progress_items" drop column "episode_number";');
+  }
+}

src/db/migrations/Migration20231111160045.ts

@@ -0,0 +1,25 @@
+import { Migration } from '@mikro-orm/migrations';
+
+export class Migration20231111160045 extends Migration {
+  async up(): Promise<void> {
+    this.addSql(
+      'alter table "provider_metrics" add column "hostname" varchar(255) not null;',
+    );
+    this.addSql(
+      'alter table "provider_metrics" alter column "error_message" type text using ("error_message"::text);',
+    );
+    this.addSql(
+      'alter table "provider_metrics" alter column "full_error" type text using ("full_error"::text);',
+    );
+  }
+
+  async down(): Promise<void> {
+    this.addSql(
+      'alter table "provider_metrics" alter column "error_message" type varchar(255) using ("error_message"::varchar(255));',
+    );
+    this.addSql(
+      'alter table "provider_metrics" alter column "full_error" type varchar(255) using ("full_error"::varchar(255));',
+    );
+    this.addSql('alter table "provider_metrics" drop column "hostname";');
+  }
+}

src/db/models/ProgressItem.ts

@@ -6,7 +6,7 @@ export const progressMetaSchema = z.object({
   title: z.string(),
   year: z.number(),
   poster: z.string().optional(),
-  type: z.string(),
+  type: z.union([z.literal('show'), z.literal('movie')]),
 });
 
 export type ProgressMeta = z.infer<typeof progressMetaSchema>;
@@ -29,6 +29,12 @@ export class ProgressItem {
   @Property({ name: 'episode_id', nullable: true })
   episodeId?: string;
 
+  @Property({ name: 'season_number', nullable: true })
+  seasonNumber?: number;
+
+  @Property({ name: 'episode_number', nullable: true })
+  episodeNumber?: number;
+
   @Property({
     name: 'meta',
     type: types.json,
@@ -47,9 +53,16 @@ export class ProgressItem {
 }
 
 export interface ProgressItemDTO {
+  id: string;
   tmdbId: string;
-  seasonId?: string;
+  season: {
-  episodeId?: string;
+    id?: string;
+    number?: number;
+  };
+  episode: {
+    id?: string;
+    number?: number;
+  };
   meta: {
     title: string;
     year: number;
@@ -65,9 +78,16 @@ export function formatProgressItem(
   progressItem: ProgressItem,
 ): ProgressItemDTO {
   return {
+    id: progressItem.id,
     tmdbId: progressItem.tmdbId,
-    seasonId: progressItem.seasonId,
+    episode: {
-    episodeId: progressItem.episodeId,
+      id: progressItem.episodeId,
+      number: progressItem.episodeNumber,
+    },
+    season: {
+      id: progressItem.seasonId,
+      number: progressItem.seasonNumber,
+    },
     meta: {
       title: progressItem.meta.title,
       year: progressItem.meta.year,

src/db/models/ProviderMetrics.ts

@@ -40,9 +40,12 @@ export class ProviderMetric {
   @Property({ name: 'embed_id', nullable: true })
   embedId?: string;
 
-  @Property({ name: 'error_message', nullable: true })
+  @Property({ name: 'error_message', nullable: true, type: 'text' })
   errorMessage?: string;
 
-  @Property({ name: 'full_error', nullable: true })
+  @Property({ name: 'full_error', nullable: true, type: 'text' })
   fullError?: string;
+
+  @Property({ name: 'hostname' })
+  hostname!: string;
 }

src/mikro-orm.config.ts

@@ -1,4 +1,4 @@
+import { ormConf } from '@/config/orm';
 import { makeOrmConfig } from '@/modules/mikro/orm';
-import { conf } from '@/config';
 
-export default makeOrmConfig(conf.postgres.connection);
+export default makeOrmConfig(ormConf.postgres.connection);

src/modules/fastify/index.ts

@@ -56,8 +56,10 @@ export async function setupFastify(): Promise<FastifyInstance> {
 
   // plugins
   log.info(`setting up plugins`, { evt: 'setup-plugins' });
+  const corsDomains = conf.server.cors.split(' ').filter((v) => v.length > 0);
+  const corsSetting = conf.server.allowAnySite ? true : corsDomains;
   await app.register(cors, {
-    origin: conf.server.cors.split(' ').filter((v) => v.length > 0),
+    origin: corsSetting,
     credentials: true,
   });
 

src/modules/metrics/index.ts

@@ -18,6 +18,7 @@ export type Metrics = {
     | 'type'
     | 'provider_id'
     | 'embed_id'
+    | 'hostname'
   >;
 };
 
@@ -57,6 +58,7 @@ export async function setupMetrics(app: FastifyInstance) {
         'tmdb_id',
         'type',
         'embed_id',
+        'hostname',
       ],
     }),
   };

src/modules/mikro/index.ts

@@ -14,8 +14,10 @@ export function getORM() {
 
 export async function setupMikroORM() {
   log.info(`Connecting to postgres`, { evt: 'connecting' });
-  const mikro = await createORM(conf.postgres.connection, (msg) =>
+  const mikro = await createORM(
-    log.info(msg),
+    conf.postgres.connection,
+    conf.postgres.debugLogging,
+    (msg) => log.info(msg),
   );
 
   if (conf.postgres.syncSchema) {

src/modules/mikro/orm.ts

@@ -16,9 +16,14 @@ export function makeOrmConfig(url: string): Options<PostgreSqlDriver> {
   };
 }
 
-export async function createORM(url: string, log: (msg: string) => void) {
+export async function createORM(
+  url: string,
+  debug: boolean,
+  log: (msg: string) => void,
+) {
   return await MikroORM.init<PostgreSqlDriver>({
     ...makeOrmConfig(url),
     logger: log,
+    debug,
   });
 }

src/routes/auth/login.ts

@@ -1,6 +1,6 @@
 import { ChallengeCode } from '@/db/models/ChallengeCode';
 import { formatSession } from '@/db/models/Session';
-import { User } from '@/db/models/User';
+import { User, formatUser } from '@/db/models/User';
 import { assertChallengeCode } from '@/services/challenge';
 import { StatusError } from '@/services/error';
 import { handle } from '@/services/handler';
@@ -85,6 +85,7 @@ export const loginAuthRouter = makeRouter((app) => {
         await em.persistAndFlush([session, user]);
 
         return {
+          user: formatUser(user),
           session: formatSession(session),
           token: makeSessionToken(session),
         };

src/routes/meta.ts

@@ -1,17 +1,21 @@
-import { conf } from '@/config';
+import { conf, version } from '@/config';
 import { handle } from '@/services/handler';
 import { makeRouter } from '@/services/router';
 
 export const metaRouter = makeRouter((app) => {
   app.get(
     '/healthcheck',
-    handle(async ({ em }) => {
+    handle(async ({ em, res }) => {
       const databaseConnected = await em.config
         .getDriver()
         .getConnection()
         .isConnected();
+
+      const healthy = databaseConnected;
+      if (!healthy) res.status(503);
+
       return {
-        healthy: databaseConnected,
+        healthy,
         databaseConnected,
       };
     }),
@@ -22,6 +26,7 @@ export const metaRouter = makeRouter((app) => {
       return {
         name: conf.meta.name,
         description: conf.meta.description,
+        version: version,
         hasCaptcha: conf.captcha.enabled,
         captchaClientKey: conf.captcha.clientKey,
       };

src/routes/metrics.ts

@@ -37,19 +37,25 @@ export const metricsRouter = makeRouter((app) => {
         window: '30m',
       });
 
+      const hostname = req.headers.origin?.slice(0, 255) ?? 'unknown origin';
+
       const entities = body.items.map((v) => {
+        const errorMessage = v.errorMessage?.slice(0, 200);
+        const truncatedFullError = v.fullError?.slice(0, 2000);
+
         const metric = new ProviderMetric();
         em.assign(metric, {
           providerId: v.providerId,
           embedId: v.embedId,
-          fullError: v.fullError,
+          fullError: truncatedFullError,
-          errorMessage: v.errorMessage,
+          errorMessage: errorMessage,
           episodeId: v.episodeId,
           seasonId: v.seasonId,
           status: v.status,
           title: v.title,
           tmdbId: v.tmdbId,
           type: v.type,
+          hostname,
         });
         return metric;
       });
@@ -63,6 +69,7 @@ export const metricsRouter = makeRouter((app) => {
           title: entity.title,
           tmdb_id: entity.tmdbId,
           type: entity.type,
+          hostname,
         });
       });
 

src/routes/users/bookmark.ts

@@ -6,8 +6,14 @@ import {
 import { StatusError } from '@/services/error';
 import { handle } from '@/services/handler';
 import { makeRouter } from '@/services/router';
+import { randomUUID } from 'crypto';
 import { z } from 'zod';
 
+const bookmarkDataSchema = z.object({
+  tmdbId: z.string(),
+  meta: bookmarkMetaSchema,
+});
+
 export const userBookmarkRouter = makeRouter((app) => {
   app.get(
     '/users/:uid/bookmarks',
@@ -40,9 +46,7 @@ export const userBookmarkRouter = makeRouter((app) => {
           uid: z.string(),
           tmdbid: z.string(),
         }),
-        body: z.object({
+        body: bookmarkDataSchema,
-          meta: bookmarkMetaSchema,
-        }),
       },
     },
     handle(async ({ auth, params, body, em }) => {
@@ -70,6 +74,40 @@ export const userBookmarkRouter = makeRouter((app) => {
     }),
   );
 
+  app.put(
+    '/users/:uid/bookmarks',
+    {
+      schema: {
+        params: z.object({
+          uid: z.string(),
+        }),
+        body: z.array(bookmarkDataSchema),
+      },
+    },
+    handle(async ({ auth, params, body, em }) => {
+      await auth.assert();
+
+      if (auth.user.id !== params.uid)
+        throw new StatusError('Cannot modify user other than yourself', 403);
+
+      const bookmarks = await em.upsertMany(
+        Bookmark,
+        body.map((item) => ({
+          userId: params.uid,
+          tmdbId: item.tmdbId,
+          meta: item.meta,
+          updatedAt: new Date(),
+        })),
+        {
+          onConflictFields: ['tmdbId', 'userId'],
+        },
+      );
+
+      await em.flush();
+      return bookmarks.map(formatBookmark);
+    }),
+  );
+
   app.delete(
     '/users/:uid/bookmarks/:tmdbid',
     {

src/routes/users/get.ts

@@ -1,3 +1,4 @@
+import { Session, formatSession } from '@/db/models/Session';
 import { User, formatUser } from '@/db/models/User';
 import { StatusError } from '@/services/error';
 import { handle } from '@/services/handler';
@@ -25,7 +26,17 @@ export const userGetRouter = makeRouter((app) => {
       const user = await em.findOne(User, { id: uid });
       if (!user) throw new StatusError('User does not exist', 404);
 
-      return formatUser(user);
+      let session: Session | undefined = undefined;
+
+      if (uid === '@me') {
+        session = (await auth.getSession()) ?? undefined;
+        if (!session) throw new StatusError('Session does not exist', 400);
+      }
+
+      return {
+        user: formatUser(user),
+        session: session ? formatSession(session) : undefined,
+      };
     }),
   );
 });

src/routes/users/progress.ts

@@ -6,8 +6,20 @@ import {
 import { StatusError } from '@/services/error';
 import { handle } from '@/services/handler';
 import { makeRouter } from '@/services/router';
+import { randomUUID } from 'crypto';
 import { z } from 'zod';
 
+const progressItemSchema = z.object({
+  meta: progressMetaSchema,
+  tmdbId: z.string(),
+  duration: z.number().transform((n) => Math.round(n)),
+  watched: z.number().transform((n) => Math.round(n)),
+  seasonId: z.string().optional(),
+  episodeId: z.string().optional(),
+  seasonNumber: z.number().optional(),
+  episodeNumber: z.number().optional(),
+});
+
 export const userProgressRouter = makeRouter((app) => {
   app.put(
     '/users/:uid/progress/:tmdbid',
@@ -17,13 +29,7 @@ export const userProgressRouter = makeRouter((app) => {
           uid: z.string(),
           tmdbid: z.string(),
         }),
-        body: z.object({
+        body: progressItemSchema,
-          meta: progressMetaSchema,
-          seasonId: z.string().optional(),
-          episodeId: z.string().optional(),
-          duration: z.number(),
-          watched: z.number(),
-        }),
       },
     },
     handle(async ({ auth, params, body, em }) => {
@@ -38,12 +44,15 @@ export const userProgressRouter = makeRouter((app) => {
         episodeId: body.episodeId,
         seasonId: body.seasonId,
       });
+
       if (!progressItem) {
         progressItem = new ProgressItem();
         progressItem.tmdbId = params.tmdbid;
         progressItem.userId = params.uid;
         progressItem.episodeId = body.episodeId;
         progressItem.seasonId = body.seasonId;
+        progressItem.episodeNumber = body.episodeNumber;
+        progressItem.seasonNumber = body.seasonNumber;
       }
 
       em.assign(progressItem, {
@@ -58,6 +67,80 @@ export const userProgressRouter = makeRouter((app) => {
     }),
   );
 
+  app.put(
+    '/users/:uid/progress/import',
+    {
+      schema: {
+        params: z.object({
+          uid: z.string(),
+        }),
+        body: z.array(progressItemSchema),
+      },
+    },
+    handle(async ({ auth, params, body, em, req, limiter }) => {
+      await auth.assert();
+
+      if (auth.user.id !== params.uid)
+        throw new StatusError('Cannot modify user other than yourself', 403);
+
+      const itemsUpserted: ProgressItem[] = [];
+
+      const newItems = [...body];
+
+      for (const existingItem of await em.find(ProgressItem, {
+        userId: params.uid,
+      })) {
+        const newItemIndex = newItems.findIndex(
+          (item) =>
+            item.tmdbId == existingItem.tmdbId &&
+            item.seasonId == existingItem.seasonId &&
+            item.episodeId == existingItem.episodeId,
+        );
+
+        if (newItemIndex > -1) {
+          const newItem = newItems[newItemIndex];
+          if (existingItem.watched < newItem.watched) {
+            existingItem.updatedAt = new Date();
+            existingItem.watched = newItem.watched;
+          }
+          itemsUpserted.push(existingItem);
+
+          // Remove the item from the array, we have processed it
+          newItems.splice(newItemIndex, 1);
+        }
+      }
+
+      // All unprocessed items, aka all items that don't already exist
+      for (const newItem of newItems) {
+        itemsUpserted.push({
+          id: randomUUID(),
+          duration: newItem.duration,
+          episodeId: newItem.episodeId,
+          episodeNumber: newItem.episodeNumber,
+          meta: newItem.meta,
+          seasonId: newItem.seasonId,
+          seasonNumber: newItem.seasonNumber,
+          tmdbId: newItem.tmdbId,
+          userId: params.uid,
+          watched: newItem.watched,
+          updatedAt: new Date(),
+        });
+      }
+
+      const progressItems = await em.upsertMany(ProgressItem, itemsUpserted);
+
+      await em.flush();
+
+      await limiter?.assertAndBump(req, {
+        id: 'progress_import',
+        max: 5,
+        window: '10m',
+      });
+
+      return progressItems.map(formatProgressItem);
+    }),
+  );
+
   app.delete(
     '/users/:uid/progress/:tmdbid',
     {

src/services/logger.ts

@@ -64,6 +64,8 @@ export function scopedLogger(service: string, meta: object = {}) {
   return logger;
 }
 
+const ignoredUrls = ['/healthcheck', '/metrics'];
+
 export function makeFastifyLogger(logger: winston.Logger) {
   logger.format = winston.format.combine(
     winston.format((info) => {
@@ -78,6 +80,9 @@ export function makeFastifyLogger(logger: winston.Logger) {
         let url = request.url;
         try {
           const pathParts = (request.url as string).split('?', 2);
+
+          if (ignoredUrls.includes(pathParts[0])) return false;
+
           if (pathParts[1]) {
             const searchParams = new URLSearchParams(pathParts[1]);
             pathParts[1] = searchParams.toString();