Merge pull request #13 from movie-web/dev

Version 1.1: Progress and Bookmark Bulk Importing

package.json

@@ -1,6 +1,6 @@
 {
   "name": "backend",
-  "version": "1.0.3",
+  "version": "1.1.0",
   "private": true,
   "homepage": "https://github.com/movie-web/backend",
   "engines": {

src/config/index.ts

@@ -8,7 +8,7 @@ const fragments = {
   dockerdev: dockerFragment,
 };
 
-export const version = '1.0.3';
+export const version = process.env.npm_package_version ?? 'unknown';
 
 export const conf = createConfigLoader()
   .addFromEnvironment('MWB_')

src/config/schema.ts

@@ -41,6 +41,10 @@ export const configSchema = z.object({
     // will always keep the database schema in sync with the connected database
     // it is extremely destructive, do not use it EVER in production
     syncSchema: z.coerce.boolean().default(false),
+
+    // Enable debug logging for MikroORM - Outputs queries and entity management logs
+    // Do NOT use in production, leaks all sensitive data
+    debugLogging: z.coerce.boolean().default(false),
   }),
   crypto: z.object({
     // session secret. used for signing session tokens

src/db/migrations/.snapshot-movie_web.json

@@ -363,20 +363,29 @@
         },
         "error_message": {
           "name": "error_message",
-          "type": "varchar(255)",
+          "type": "text",
           "unsigned": false,
           "autoincrement": false,
           "primary": false,
           "nullable": true,
-          "mappedType": "string"
+          "mappedType": "text"
         },
         "full_error": {
           "name": "full_error",
-          "type": "varchar(255)",
+          "type": "text",
           "unsigned": false,
           "autoincrement": false,
           "primary": false,
           "nullable": true,
+          "mappedType": "text"
+        },
+        "hostname": {
+          "name": "hostname",
+          "type": "varchar(255)",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": false,
           "mappedType": "string"
         }
       },

src/db/migrations/Migration20231111160045.ts

@@ -0,0 +1,25 @@
+import { Migration } from '@mikro-orm/migrations';
+
+export class Migration20231111160045 extends Migration {
+  async up(): Promise<void> {
+    this.addSql(
+      'alter table "provider_metrics" add column "hostname" varchar(255) not null;',
+    );
+    this.addSql(
+      'alter table "provider_metrics" alter column "error_message" type text using ("error_message"::text);',
+    );
+    this.addSql(
+      'alter table "provider_metrics" alter column "full_error" type text using ("full_error"::text);',
+    );
+  }
+
+  async down(): Promise<void> {
+    this.addSql(
+      'alter table "provider_metrics" alter column "error_message" type varchar(255) using ("error_message"::varchar(255));',
+    );
+    this.addSql(
+      'alter table "provider_metrics" alter column "full_error" type varchar(255) using ("full_error"::varchar(255));',
+    );
+    this.addSql('alter table "provider_metrics" drop column "hostname";');
+  }
+}

src/db/models/ProgressItem.ts

@@ -53,6 +53,7 @@ export class ProgressItem {
 }
 
 export interface ProgressItemDTO {
+  id: string;
   tmdbId: string;
   season: {
     id?: string;
@@ -77,6 +78,7 @@ export function formatProgressItem(
   progressItem: ProgressItem,
 ): ProgressItemDTO {
   return {
+    id: progressItem.id,
     tmdbId: progressItem.tmdbId,
     episode: {
       id: progressItem.episodeId,

src/db/models/ProviderMetrics.ts

@@ -40,9 +40,12 @@ export class ProviderMetric {
   @Property({ name: 'embed_id', nullable: true })
   embedId?: string;
 
-  @Property({ name: 'error_message', nullable: true })
+  @Property({ name: 'error_message', nullable: true, type: 'text' })
   errorMessage?: string;
 
-  @Property({ name: 'full_error', nullable: true })
+  @Property({ name: 'full_error', nullable: true, type: 'text' })
   fullError?: string;
+
+  @Property({ name: 'hostname' })
+  hostname!: string;
 }

src/modules/metrics/index.ts

@@ -18,6 +18,7 @@ export type Metrics = {
     | 'type'
     | 'provider_id'
     | 'embed_id'
+    | 'hostname'
   >;
 };
 
@@ -57,6 +58,7 @@ export async function setupMetrics(app: FastifyInstance) {
         'tmdb_id',
         'type',
         'embed_id',
+        'hostname',
       ],
     }),
   };

src/modules/mikro/index.ts

@@ -14,8 +14,10 @@ export function getORM() {
 
 export async function setupMikroORM() {
   log.info(`Connecting to postgres`, { evt: 'connecting' });
-  const mikro = await createORM(conf.postgres.connection, (msg) =>
-    log.info(msg),
+  const mikro = await createORM(
+    conf.postgres.connection,
+    conf.postgres.debugLogging,
+    (msg) => log.info(msg),
   );
 
   if (conf.postgres.syncSchema) {

src/modules/mikro/orm.ts

@@ -16,9 +16,14 @@ export function makeOrmConfig(url: string): Options<PostgreSqlDriver> {
   };
 }
 
-export async function createORM(url: string, log: (msg: string) => void) {
+export async function createORM(
+  url: string,
+  debug: boolean,
+  log: (msg: string) => void,
+) {
   return await MikroORM.init<PostgreSqlDriver>({
     ...makeOrmConfig(url),
     logger: log,
+    debug,
   });
 }

src/routes/metrics.ts

@@ -37,19 +37,25 @@ export const metricsRouter = makeRouter((app) => {
         window: '30m',
       });
 
+      const hostname = req.headers.origin?.slice(0, 255) ?? 'unknown origin';
+
       const entities = body.items.map((v) => {
+        const errorMessage = v.errorMessage?.slice(0, 200);
+        const truncatedFullError = v.fullError?.slice(0, 2000);
+
         const metric = new ProviderMetric();
         em.assign(metric, {
           providerId: v.providerId,
           embedId: v.embedId,
-          fullError: v.fullError,
-          errorMessage: v.errorMessage,
+          fullError: truncatedFullError,
+          errorMessage: errorMessage,
           episodeId: v.episodeId,
           seasonId: v.seasonId,
           status: v.status,
           title: v.title,
           tmdbId: v.tmdbId,
           type: v.type,
+          hostname,
         });
         return metric;
       });
@@ -63,6 +69,7 @@ export const metricsRouter = makeRouter((app) => {
           title: entity.title,
           tmdb_id: entity.tmdbId,
           type: entity.type,
+          hostname,
         });
       });
 

src/routes/users/bookmark.ts

@@ -6,8 +6,14 @@ import {
 import { StatusError } from '@/services/error';
 import { handle } from '@/services/handler';
 import { makeRouter } from '@/services/router';
+import { randomUUID } from 'crypto';
 import { z } from 'zod';
 
+const bookmarkDataSchema = z.object({
+  tmdbId: z.string(),
+  meta: bookmarkMetaSchema,
+});
+
 export const userBookmarkRouter = makeRouter((app) => {
   app.get(
     '/users/:uid/bookmarks',
@@ -40,9 +46,7 @@ export const userBookmarkRouter = makeRouter((app) => {
           uid: z.string(),
           tmdbid: z.string(),
         }),
-        body: z.object({
-          meta: bookmarkMetaSchema,
-        }),
+        body: bookmarkDataSchema,
       },
     },
     handle(async ({ auth, params, body, em }) => {
@@ -70,6 +74,40 @@ export const userBookmarkRouter = makeRouter((app) => {
     }),
   );
 
+  app.put(
+    '/users/:uid/bookmarks',
+    {
+      schema: {
+        params: z.object({
+          uid: z.string(),
+        }),
+        body: z.array(bookmarkDataSchema),
+      },
+    },
+    handle(async ({ auth, params, body, em }) => {
+      await auth.assert();
+
+      if (auth.user.id !== params.uid)
+        throw new StatusError('Cannot modify user other than yourself', 403);
+
+      const bookmarks = await em.upsertMany(
+        Bookmark,
+        body.map((item) => ({
+          userId: params.uid,
+          tmdbId: item.tmdbId,
+          meta: item.meta,
+          updatedAt: new Date(),
+        })),
+        {
+          onConflictFields: ['tmdbId', 'userId'],
+        },
+      );
+
+      await em.flush();
+      return bookmarks.map(formatBookmark);
+    }),
+  );
+
   app.delete(
     '/users/:uid/bookmarks/:tmdbid',
     {

src/routes/users/get.ts

@@ -1,3 +1,4 @@
+import { Session, formatSession } from '@/db/models/Session';
 import { User, formatUser } from '@/db/models/User';
 import { StatusError } from '@/services/error';
 import { handle } from '@/services/handler';
@@ -25,7 +26,17 @@ export const userGetRouter = makeRouter((app) => {
       const user = await em.findOne(User, { id: uid });
       if (!user) throw new StatusError('User does not exist', 404);
 
-      return formatUser(user);
+      let session: Session | undefined = undefined;
+
+      if (uid === '@me') {
+        session = (await auth.getSession()) ?? undefined;
+        if (!session) throw new StatusError('Session does not exist', 400);
+      }
+
+      return {
+        user: formatUser(user),
+        session: session ? formatSession(session) : undefined,
+      };
     }),
   );
 });

src/routes/users/progress.ts

@@ -6,8 +6,20 @@ import {
 import { StatusError } from '@/services/error';
 import { handle } from '@/services/handler';
 import { makeRouter } from '@/services/router';
+import { randomUUID } from 'crypto';
 import { z } from 'zod';
 
+const progressItemSchema = z.object({
+  meta: progressMetaSchema,
+  tmdbId: z.string(),
+  duration: z.number().transform((n) => Math.round(n)),
+  watched: z.number().transform((n) => Math.round(n)),
+  seasonId: z.string().optional(),
+  episodeId: z.string().optional(),
+  seasonNumber: z.number().optional(),
+  episodeNumber: z.number().optional(),
+});
+
 export const userProgressRouter = makeRouter((app) => {
   app.put(
     '/users/:uid/progress/:tmdbid',
@@ -17,15 +29,7 @@ export const userProgressRouter = makeRouter((app) => {
           uid: z.string(),
           tmdbid: z.string(),
         }),
-        body: z.object({
-          meta: progressMetaSchema,
-          duration: z.number(),
-          watched: z.number(),
-          seasonId: z.string().optional(),
-          episodeId: z.string().optional(),
-          seasonNumber: z.number().optional(),
-          episodeNumber: z.number().optional(),
-        }),
+        body: progressItemSchema,
       },
     },
     handle(async ({ auth, params, body, em }) => {
@@ -63,6 +67,80 @@ export const userProgressRouter = makeRouter((app) => {
     }),
   );
 
+  app.put(
+    '/users/:uid/progress/import',
+    {
+      schema: {
+        params: z.object({
+          uid: z.string(),
+        }),
+        body: z.array(progressItemSchema),
+      },
+    },
+    handle(async ({ auth, params, body, em, req, limiter }) => {
+      await auth.assert();
+
+      if (auth.user.id !== params.uid)
+        throw new StatusError('Cannot modify user other than yourself', 403);
+
+      const itemsUpserted: ProgressItem[] = [];
+
+      const newItems = [...body];
+
+      for (const existingItem of await em.find(ProgressItem, {
+        userId: params.uid,
+      })) {
+        const newItemIndex = newItems.findIndex(
+          (item) =>
+            item.tmdbId == existingItem.tmdbId &&
+            item.seasonId == existingItem.seasonId &&
+            item.episodeId == existingItem.episodeId,
+        );
+
+        if (newItemIndex > -1) {
+          const newItem = newItems[newItemIndex];
+          if (existingItem.watched < newItem.watched) {
+            existingItem.updatedAt = new Date();
+            existingItem.watched = newItem.watched;
+          }
+          itemsUpserted.push(existingItem);
+
+          // Remove the item from the array, we have processed it
+          newItems.splice(newItemIndex, 1);
+        }
+      }
+
+      // All unprocessed items, aka all items that don't already exist
+      for (const newItem of newItems) {
+        itemsUpserted.push({
+          id: randomUUID(),
+          duration: newItem.duration,
+          episodeId: newItem.episodeId,
+          episodeNumber: newItem.episodeNumber,
+          meta: newItem.meta,
+          seasonId: newItem.seasonId,
+          seasonNumber: newItem.seasonNumber,
+          tmdbId: newItem.tmdbId,
+          userId: params.uid,
+          watched: newItem.watched,
+          updatedAt: new Date(),
+        });
+      }
+
+      const progressItems = await em.upsertMany(ProgressItem, itemsUpserted);
+
+      await em.flush();
+
+      await limiter?.assertAndBump(req, {
+        id: 'progress_import',
+        max: 5,
+        window: '10m',
+      });
+
+      return progressItems.map(formatProgressItem);
+    }),
+  );
+
   app.delete(
     '/users/:uid/progress/:tmdbid',
     {