From 8e10eb7d591d5d3d92be90b3fa95613760cf1c8b Mon Sep 17 00:00:00 2001 From: William Oldham Date: Sat, 18 Nov 2023 19:21:02 +0000 Subject: [PATCH 1/9] Add option to enable MikroORM database debug logging --- src/config/schema.ts | 4 ++++ src/modules/mikro/index.ts | 6 ++++-- src/modules/mikro/orm.ts | 7 ++++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/config/schema.ts b/src/config/schema.ts index b94e46c..e6ec194 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -41,6 +41,10 @@ export const configSchema = z.object({ // will always keep the database schema in sync with the connected database // it is extremely destructive, do not use it EVER in production syncSchema: z.coerce.boolean().default(false), + + // Enable debug logging for MikroORM - Outputs queries and entity management logs + // Do NOT use in production, leaks all sensitive data + debugLogging: z.coerce.boolean().default(false), }), crypto: z.object({ // session secret. used for signing session tokens diff --git a/src/modules/mikro/index.ts b/src/modules/mikro/index.ts index d9dae43..415b835 100644 --- a/src/modules/mikro/index.ts +++ b/src/modules/mikro/index.ts @@ -14,8 +14,10 @@ export function getORM() { export async function setupMikroORM() { log.info(`Connecting to postgres`, { evt: 'connecting' }); - const mikro = await createORM(conf.postgres.connection, (msg) => - log.info(msg), + const mikro = await createORM( + conf.postgres.connection, + conf.postgres.debugLogging, + (msg) => log.info(msg), ); if (conf.postgres.syncSchema) { diff --git a/src/modules/mikro/orm.ts b/src/modules/mikro/orm.ts index a22cc67..3987b46 100644 --- a/src/modules/mikro/orm.ts +++ b/src/modules/mikro/orm.ts @@ -16,9 +16,14 @@ export function makeOrmConfig(url: string): Options { }; } -export async function createORM(url: string, log: (msg: string) => void) { +export async function createORM( + url: string, + debug: boolean, + log: (msg: string) => void, +) { return await MikroORM.init({ ...makeOrmConfig(url), logger: log, + debug, }); } From 6d200e85682c4b746588e7e8a3c6e28035302a34 Mon Sep 17 00:00:00 2001 From: William Oldham Date: Sat, 18 Nov 2023 19:21:24 +0000 Subject: [PATCH 2/9] Return ID in progress item response --- src/db/models/ProgressItem.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/db/models/ProgressItem.ts b/src/db/models/ProgressItem.ts index aad2070..ab45476 100644 --- a/src/db/models/ProgressItem.ts +++ b/src/db/models/ProgressItem.ts @@ -53,6 +53,7 @@ export class ProgressItem { } export interface ProgressItemDTO { + id: string; tmdbId: string; season: { id?: string; @@ -77,6 +78,7 @@ export function formatProgressItem( progressItem: ProgressItem, ): ProgressItemDTO { return { + id: progressItem.id, tmdbId: progressItem.tmdbId, episode: { id: progressItem.episodeId, From a73baf555bd84543c8164651b4b26eca8f4ee909 Mon Sep 17 00:00:00 2001 From: William Oldham Date: Sat, 18 Nov 2023 19:21:52 +0000 Subject: [PATCH 3/9] Add progress importing endpoint --- src/routes/users/progress.ts | 99 ++++++++++++++++++++++++++++++++---- 1 file changed, 90 insertions(+), 9 deletions(-) diff --git a/src/routes/users/progress.ts b/src/routes/users/progress.ts index 37f5797..1ff5221 100644 --- a/src/routes/users/progress.ts +++ b/src/routes/users/progress.ts @@ -6,8 +6,20 @@ import { import { StatusError } from '@/services/error'; import { handle } from '@/services/handler'; import { makeRouter } from '@/services/router'; +import { randomUUID } from 'crypto'; import { z } from 'zod'; +const progressItemSchema = z.object({ + meta: progressMetaSchema, + tmdbId: z.string(), + duration: z.number(), + watched: z.number(), + seasonId: z.string().optional(), + episodeId: z.string().optional(), + seasonNumber: z.number().optional(), + episodeNumber: z.number().optional(), +}); + export const userProgressRouter = makeRouter((app) => { app.put( '/users/:uid/progress/:tmdbid', @@ -17,15 +29,7 @@ export const userProgressRouter = makeRouter((app) => { uid: z.string(), tmdbid: z.string(), }), - body: z.object({ - meta: progressMetaSchema, - duration: z.number(), - watched: z.number(), - seasonId: z.string().optional(), - episodeId: z.string().optional(), - seasonNumber: z.number().optional(), - episodeNumber: z.number().optional(), - }), + body: progressItemSchema, }, }, handle(async ({ auth, params, body, em }) => { @@ -63,6 +67,83 @@ export const userProgressRouter = makeRouter((app) => { }), ); + app.put( + '/users/:uid/progress/import', + { + schema: { + params: z.object({ + uid: z.string(), + }), + body: z.array(progressItemSchema), + }, + }, + handle(async ({ auth, params, body: newItems, em, req, limiter }) => { + await auth.assert(); + + if (auth.user.id !== params.uid) + throw new StatusError('Cannot modify user other than yourself', 403); + + const existingItems = await em.find(ProgressItem, { userId: params.uid }); + + for (const newItem of newItems) { + const existingItem = existingItems.find( + (item) => + item.tmdbId == newItem.tmdbId && + item.seasonId == newItem.seasonId && + item.episodeId == newItem.episodeId, + ); + + if (existingItem) { + if (existingItem.watched < newItem.watched) { + existingItem.updatedAt = new Date(); + existingItem.watched = newItem.watched; + } + continue; + } + + existingItems.push({ + id: randomUUID(), + duration: newItem.duration, + episodeId: newItem.episodeId, + episodeNumber: newItem.episodeNumber, + meta: newItem.meta, + seasonId: newItem.seasonId, + seasonNumber: newItem.seasonNumber, + tmdbId: newItem.tmdbId, + userId: params.uid, + watched: newItem.watched, + updatedAt: new Date(), + }); + } + + const progressItems = await em.upsertMany(ProgressItem, existingItems); + + await em.flush(); + + await limiter?.assertAndBump(req, { + id: 'progress_import', + max: 5, + window: '10m', + }); + + // Construct a response that only has the items that were requested to be updated in the same order + // ! is used on find as the item *should* always exist if the code above works correctly + const newItemResponses = newItems + .map( + (newItem) => + progressItems.find( + (item) => + item.tmdbId == newItem.tmdbId && + item.seasonId == newItem.seasonId && + item.episodeId == newItem.episodeId, + )!, + ) + .map(formatProgressItem); + + return newItemResponses; + }), + ); + app.delete( '/users/:uid/progress/:tmdbid', { From 3643eea655ba3414f544506673d5ea159f1c5293 Mon Sep 17 00:00:00 2001 From: William Oldham Date: Sat, 18 Nov 2023 19:25:33 +0000 Subject: [PATCH 4/9] Update progress import endpoint --- src/routes/users/progress.ts | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/src/routes/users/progress.ts b/src/routes/users/progress.ts index 1ff5221..b9916a4 100644 --- a/src/routes/users/progress.ts +++ b/src/routes/users/progress.ts @@ -84,6 +84,7 @@ export const userProgressRouter = makeRouter((app) => { throw new StatusError('Cannot modify user other than yourself', 403); const existingItems = await em.find(ProgressItem, { userId: params.uid }); + const itemsToUpsert: ProgressItem[] = []; for (const newItem of newItems) { const existingItem = existingItems.find( @@ -98,10 +99,11 @@ export const userProgressRouter = makeRouter((app) => { existingItem.updatedAt = new Date(); existingItem.watched = newItem.watched; } + itemsToUpsert.push(existingItem); continue; } - existingItems.push({ + itemsToUpsert.push({ id: randomUUID(), duration: newItem.duration, episodeId: newItem.episodeId, @@ -116,7 +118,7 @@ export const userProgressRouter = makeRouter((app) => { }); } - const progressItems = await em.upsertMany(ProgressItem, existingItems); + const progressItems = await em.upsertMany(ProgressItem, itemsToUpsert); await em.flush(); @@ -126,21 +128,7 @@ export const userProgressRouter = makeRouter((app) => { window: '10m', }); - // Construct a response that only has the items that were requested to be updated in the same order - // ! is used on find as the item *should* always exist if the code above works correctly - const newItemResponses = newItems - .map( - (newItem) => - progressItems.find( - (item) => - item.tmdbId == newItem.tmdbId && - item.seasonId == newItem.seasonId && - item.episodeId == newItem.episodeId, - )!, - ) - .map(formatProgressItem); - - return newItemResponses; + return progressItems.map(formatProgressItem); }), ); From 787c8a96d61edffa149e862e667ff9a1cdf84b54 Mon Sep 17 00:00:00 2001 From: William Oldham Date: Sat, 18 Nov 2023 19:48:07 +0000 Subject: [PATCH 5/9] Update progress import endpoint to be more efficient on memory --- src/routes/users/progress.ts | 37 ++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/src/routes/users/progress.ts b/src/routes/users/progress.ts index b9916a4..ed8e651 100644 --- a/src/routes/users/progress.ts +++ b/src/routes/users/progress.ts @@ -77,33 +77,42 @@ export const userProgressRouter = makeRouter((app) => { body: z.array(progressItemSchema), }, }, - handle(async ({ auth, params, body: newItems, em, req, limiter }) => { + handle(async ({ auth, params, body, em, req, limiter }) => { await auth.assert(); if (auth.user.id !== params.uid) throw new StatusError('Cannot modify user other than yourself', 403); - const existingItems = await em.find(ProgressItem, { userId: params.uid }); - const itemsToUpsert: ProgressItem[] = []; + const itemsUpserted: ProgressItem[] = []; - for (const newItem of newItems) { - const existingItem = existingItems.find( + const newItems = [...body]; + + for (const existingItem of await em.find(ProgressItem, { + userId: params.uid, + })) { + const newItemIndex = newItems.findIndex( (item) => - item.tmdbId == newItem.tmdbId && - item.seasonId == newItem.seasonId && - item.episodeId == newItem.episodeId, + item.tmdbId == existingItem.tmdbId && + item.seasonId == existingItem.seasonId && + item.episodeId == existingItem.episodeId, ); - if (existingItem) { + if (newItemIndex > -1) { + const newItem = newItems[newItemIndex]; if (existingItem.watched < newItem.watched) { existingItem.updatedAt = new Date(); existingItem.watched = newItem.watched; } - itemsToUpsert.push(existingItem); - continue; - } + itemsUpserted.push(existingItem); - itemsToUpsert.push({ + // Remove the item from the array, we have processed it + newItems.splice(newItemIndex, 1); + } + } + + // All unprocessed items, aka all items that don't already exist + for (const newItem of newItems) { + itemsUpserted.push({ id: randomUUID(), duration: newItem.duration, episodeId: newItem.episodeId, @@ -118,7 +127,7 @@ export const userProgressRouter = makeRouter((app) => { }); } - const progressItems = await em.upsertMany(ProgressItem, itemsToUpsert); + const progressItems = await em.upsertMany(ProgressItem, itemsUpserted); await em.flush(); From f54b4f6553c57c37d4885785798bb92e10f44204 Mon Sep 17 00:00:00 2001 From: William Oldham Date: Sat, 18 Nov 2023 19:57:03 +0000 Subject: [PATCH 6/9] Add bookmark import endpoint --- src/routes/users/bookmark.ts | 44 +++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/src/routes/users/bookmark.ts b/src/routes/users/bookmark.ts index 451cae2..4f2b909 100644 --- a/src/routes/users/bookmark.ts +++ b/src/routes/users/bookmark.ts @@ -6,8 +6,14 @@ import { import { StatusError } from '@/services/error'; import { handle } from '@/services/handler'; import { makeRouter } from '@/services/router'; +import { randomUUID } from 'crypto'; import { z } from 'zod'; +const bookmarkDataSchema = z.object({ + tmdbId: z.string(), + meta: bookmarkMetaSchema, +}); + export const userBookmarkRouter = makeRouter((app) => { app.get( '/users/:uid/bookmarks', @@ -40,9 +46,7 @@ export const userBookmarkRouter = makeRouter((app) => { uid: z.string(), tmdbid: z.string(), }), - body: z.object({ - meta: bookmarkMetaSchema, - }), + body: bookmarkDataSchema, }, }, handle(async ({ auth, params, body, em }) => { @@ -70,6 +74,40 @@ export const userBookmarkRouter = makeRouter((app) => { }), ); + app.put( + '/users/:uid/bookmarks', + { + schema: { + params: z.object({ + uid: z.string(), + }), + body: z.array(bookmarkDataSchema), + }, + }, + handle(async ({ auth, params, body, em }) => { + await auth.assert(); + + if (auth.user.id !== params.uid) + throw new StatusError('Cannot modify user other than yourself', 403); + + const bookmarks = await em.upsertMany( + Bookmark, + body.map((item) => ({ + userId: params.uid, + tmdbId: item.tmdbId, + meta: item.meta, + updatedAt: new Date(), + })), + { + onConflictFields: ['tmdbId', 'userId'], + }, + ); + + await em.flush(); + return bookmarks.map(formatBookmark); + }), + ); + app.delete( '/users/:uid/bookmarks/:tmdbid', { From 449fd7b153ff590a9f10f267532d4624fa81487c Mon Sep 17 00:00:00 2001 From: William Oldham Date: Sat, 18 Nov 2023 20:03:52 +0000 Subject: [PATCH 7/9] Bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 53d6840..9f6c6bf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "backend", - "version": "1.0.5", + "version": "1.1.0", "private": true, "homepage": "https://github.com/movie-web/backend", "engines": { From fd9b61061c9c7bdc81b7a95c17f901eee81e3221 Mon Sep 17 00:00:00 2001 From: William Oldham Date: Sun, 19 Nov 2023 19:41:47 +0000 Subject: [PATCH 8/9] Return session for @me endpoint --- src/routes/users/get.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/routes/users/get.ts b/src/routes/users/get.ts index c6eaaf6..9ecd68d 100644 --- a/src/routes/users/get.ts +++ b/src/routes/users/get.ts @@ -1,3 +1,4 @@ +import { Session, formatSession } from '@/db/models/Session'; import { User, formatUser } from '@/db/models/User'; import { StatusError } from '@/services/error'; import { handle } from '@/services/handler'; @@ -25,7 +26,17 @@ export const userGetRouter = makeRouter((app) => { const user = await em.findOne(User, { id: uid }); if (!user) throw new StatusError('User does not exist', 404); - return formatUser(user); + let session: Session | undefined = undefined; + + if (uid === '@me') { + session = (await auth.getSession()) ?? undefined; + if (!session) throw new StatusError('Session does not exist', 400); + } + + return { + user: formatUser(user), + session: session ? formatSession(session) : undefined, + }; }), ); }); From e2b8f123b377146509b74d4d49a87e611966506a Mon Sep 17 00:00:00 2001 From: William Oldham Date: Sun, 19 Nov 2023 19:41:59 +0000 Subject: [PATCH 9/9] Round duration and watched for progress endpoints --- src/routes/users/progress.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/users/progress.ts b/src/routes/users/progress.ts index ed8e651..7575248 100644 --- a/src/routes/users/progress.ts +++ b/src/routes/users/progress.ts @@ -12,8 +12,8 @@ import { z } from 'zod'; const progressItemSchema = z.object({ meta: progressMetaSchema, tmdbId: z.string(), - duration: z.number(), - watched: z.number(), + duration: z.number().transform((n) => Math.round(n)), + watched: z.number().transform((n) => Math.round(n)), seasonId: z.string().optional(), episodeId: z.string().optional(), seasonNumber: z.number().optional(),