diff --git a/README.md b/README.md index 96dda15..e5bf090 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Backend for movie-web - [x] provider metrics - [x] cleanup old metrics in DB - [x] endpoint to consume and store metrics - - [ ] pass metrics to prometheus + - [x] pass metrics to prometheus ## Second todo list - [X] think of privacy centric method of auth diff --git a/package.json b/package.json index 0f05c42..79d2e99 100644 --- a/package.json +++ b/package.json @@ -38,10 +38,12 @@ "@mikro-orm/core": "^5.9.0", "@mikro-orm/postgresql": "^5.9.0", "@types/ms": "^0.7.33", + "async-ratelimiter": "^1.3.12", "cron": "^3.1.5", "fastify": "^4.21.0", - "fastify-metrics": "^10.3.2", + "fastify-metrics": "^10.3.3", "fastify-type-provider-zod": "^1.1.9", + "ioredis": "^5.3.2", "jsonwebtoken": "^9.0.2", "ms": "^2.1.3", "nanoid": "^3.3.6", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1acccc8..c2e233c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,6 +17,9 @@ dependencies: '@types/ms': specifier: ^0.7.33 version: 0.7.33 + async-ratelimiter: + specifier: ^1.3.12 + version: 1.3.12 cron: specifier: ^3.1.5 version: 3.1.5 @@ -24,11 +27,14 @@ dependencies: specifier: ^4.21.0 version: 4.24.3 fastify-metrics: - specifier: ^10.3.2 - version: 10.3.2(fastify@4.24.3) + specifier: ^10.3.3 + version: 10.3.3(fastify@4.24.3) fastify-type-provider-zod: specifier: ^1.1.9 version: 1.1.9(fastify@4.24.3)(zod@3.22.4) + ioredis: + specifier: ^5.3.2 + version: 5.3.2 jsonwebtoken: specifier: ^9.0.2 version: 9.0.2 @@ -230,6 +236,10 @@ packages: resolution: {integrity: sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw==} dev: true + /@ioredis/commands@1.2.0: + resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==} + dev: false + /@isaacs/cliui@8.0.2: resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} @@ -741,6 +751,11 @@ packages: resolution: {integrity: sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==} engines: {node: '>=8'} + /async-ratelimiter@1.3.12: + resolution: {integrity: sha512-W7WWxWMjJ+XEZCyQhEWGrskqDgz3k2UWM/aUlatSl3ejFLwpM/G90AYSgkHHXeY2S53fiP204GITnmIxrJMsSQ==} + engines: {node: '>= 8'} + dev: false + /async@3.2.4: resolution: {integrity: sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==} dev: false @@ -859,6 +874,11 @@ packages: fsevents: 2.3.3 dev: true + /cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + dev: false + /color-convert@1.9.3: resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==} dependencies: @@ -998,6 +1018,11 @@ packages: engines: {node: '>=12'} dev: true + /denque@2.1.0: + resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} + engines: {node: '>=0.10'} + dev: false + /diff@4.0.2: resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==} engines: {node: '>=0.3.1'} @@ -1280,8 +1305,8 @@ packages: resolution: {integrity: sha512-cIusKBIt/R/oI6z/1nyfe2FvGKVTohVRfvkOhvx0nCEW+xf5NoCXjAHcWp93uOUBchzYcsvPlrapAdX1uW+YGg==} dev: false - /fastify-metrics@10.3.2(fastify@4.24.3): - resolution: {integrity: sha512-02SEIGH02zfguqRMho0LB8L7YVAj5cIgWM0iqZslIErqaUWc1iHVAOC+YXYG3S2DZU6VHdFaMyuxjEOCQHAETA==} + /fastify-metrics@10.3.3(fastify@4.24.3): + resolution: {integrity: sha512-TmMcfrMWBSbA7yk31tFtJnWKtNXLSO7jmTRIjPX9HKC4pLmyd0JnOQ3r9XCYnev6NL9/eVRXxNfrsqQdKTLZkw==} peerDependencies: fastify: '>=4' dependencies: @@ -1565,6 +1590,23 @@ packages: engines: {node: '>= 0.10'} dev: false + /ioredis@5.3.2: + resolution: {integrity: sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==} + engines: {node: '>=12.22.0'} + dependencies: + '@ioredis/commands': 1.2.0 + cluster-key-slot: 1.1.2 + debug: 4.3.4 + denque: 2.1.0 + lodash.defaults: 4.2.0 + lodash.isarguments: 3.1.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color + dev: false + /ipaddr.js@1.9.1: resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} engines: {node: '>= 0.10'} @@ -1818,10 +1860,18 @@ packages: p-locate: 5.0.0 dev: true + /lodash.defaults@4.2.0: + resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==} + dev: false + /lodash.includes@4.3.0: resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==} dev: false + /lodash.isarguments@3.1.0: + resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==} + dev: false + /lodash.isboolean@3.0.3: resolution: {integrity: sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==} dev: false @@ -2380,6 +2430,18 @@ packages: resolve: 1.22.8 dev: false + /redis-errors@1.2.0: + resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} + engines: {node: '>=4'} + dev: false + + /redis-parser@3.0.0: + resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} + engines: {node: '>=4'} + dependencies: + redis-errors: 1.2.0 + dev: false + /reflect-metadata@0.1.13: resolution: {integrity: sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg==} dev: false @@ -2536,6 +2598,10 @@ packages: resolution: {integrity: sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==} dev: false + /standard-as-callback@2.1.0: + resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + dev: false + /string-width@4.2.3: resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} engines: {node: '>=8'} diff --git a/src/config/schema.ts b/src/config/schema.ts index be13a88..602807d 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -59,4 +59,11 @@ export const configSchema = z.object({ clientKey: z.string().min(1).optional(), }) .default({}), + ratelimits: z + .object({ + // enabled captchas on register + enabled: z.coerce.boolean().default(false), + redisUrl: z.string().optional(), + }) + .default({}), }); diff --git a/src/main.ts b/src/main.ts index 697e168..3f83d27 100644 --- a/src/main.ts +++ b/src/main.ts @@ -2,6 +2,7 @@ import { setupFastify, startFastify } from '@/modules/fastify'; import { setupJobs } from '@/modules/jobs'; import { setupMetrics } from '@/modules/metrics'; import { setupMikroORM } from '@/modules/mikro'; +import { setupRatelimits } from '@/modules/ratelimits'; import { scopedLogger } from '@/services/logger'; const log = scopedLogger('mw-backend'); @@ -11,6 +12,7 @@ async function bootstrap(): Promise { evt: 'setup', }); + await setupRatelimits(); const app = await setupFastify(); await setupMikroORM(); await setupMetrics(app); diff --git a/src/modules/ratelimits/index.ts b/src/modules/ratelimits/index.ts new file mode 100644 index 0000000..4b6a9ba --- /dev/null +++ b/src/modules/ratelimits/index.ts @@ -0,0 +1,24 @@ +import { conf } from '@/config'; +import { Limiter } from '@/modules/ratelimits/limiter'; +import { connectRedis } from '@/modules/ratelimits/redis'; +import { scopedLogger } from '@/services/logger'; + +const log = scopedLogger('ratelimits'); + +let limiter: null | Limiter = null; + +export function getLimiter() { + return limiter; +} + +export async function setupRatelimits() { + if (!conf.ratelimits.enabled) { + log.warn('Ratelimits disabled!'); + return; + } + const redis = await connectRedis(); + limiter = new Limiter({ + redis, + }); + log.info('Ratelimits have been setup!'); +} diff --git a/src/modules/ratelimits/limiter.ts b/src/modules/ratelimits/limiter.ts new file mode 100644 index 0000000..628ce59 --- /dev/null +++ b/src/modules/ratelimits/limiter.ts @@ -0,0 +1,63 @@ +import Redis from 'ioredis'; +import RateLimiter from 'async-ratelimiter'; +import ms from 'ms'; +import { StatusError } from '@/services/error'; + +export interface LimiterOptions { + redis: Redis; +} + +interface LimitBucket { + limiter: RateLimiter; +} + +interface BucketOptions { + id: string; + window: string; + max: number; + inc?: number; +} + +export class Limiter { + private redis: Redis; + private buckets: Record = {}; + + constructor(ops: LimiterOptions) { + this.redis = ops.redis; + } + + async bump(req: { ip: string }, ops: BucketOptions) { + const ip = req.ip; + if (!this.buckets[ops.id]) { + this.buckets[ops.id] = { + limiter: new RateLimiter({ + db: this.redis, + namespace: `RATELIMIT_${ops.id}`, + duration: ms(ops.window), + max: ops.max, + }), + }; + } + + for (let i = 1; i < (ops.inc ?? 0); i++) { + await this.buckets[ops.id].limiter.get({ + id: ip, + }); + } + const currentLimit = await this.buckets[ops.id].limiter.get({ + id: ip, + }); + + return { + hasBeenLimited: currentLimit.remaining <= 0, + limit: currentLimit, + }; + } + + async assertAndBump(req: { ip: string }, ops: BucketOptions) { + const { hasBeenLimited } = await this.bump(req, ops); + if (hasBeenLimited) { + throw new StatusError('Ratelimited', 429); + } + } +} diff --git a/src/modules/ratelimits/redis.ts b/src/modules/ratelimits/redis.ts new file mode 100644 index 0000000..60ed4e8 --- /dev/null +++ b/src/modules/ratelimits/redis.ts @@ -0,0 +1,7 @@ +import { conf } from '@/config'; +import Redis from 'ioredis'; + +export function connectRedis() { + if (!conf.ratelimits.redisUrl) throw new Error('missing redis URL'); + return new Redis(conf.ratelimits.redisUrl); +} diff --git a/src/routes/auth/login.ts b/src/routes/auth/login.ts index 0b9798d..1902863 100644 --- a/src/routes/auth/login.ts +++ b/src/routes/auth/login.ts @@ -25,7 +25,13 @@ export const loginAuthRouter = makeRouter((app) => { app.post( '/auth/login/start', { schema: { body: startSchema } }, - handle(async ({ em, body }) => { + handle(async ({ em, body, limiter, req }) => { + await limiter?.assertAndBump(req, { + id: 'login_challenge_tokens', + max: 20, + window: '10m', + }); + const user = await em.findOne(User, { publicKey: body.publicKey }); if (user == null) { @@ -46,7 +52,13 @@ export const loginAuthRouter = makeRouter((app) => { app.post( '/auth/login/complete', { schema: { body: completeSchema } }, - handle(async ({ em, body, req }) => { + handle(async ({ em, body, req, limiter }) => { + await limiter?.assertAndBump(req, { + id: 'login_complete', + max: 20, + window: '10m', + }); + await assertChallengeCode( em, body.challenge.code, diff --git a/src/routes/auth/manage.ts b/src/routes/auth/manage.ts index 22927ac..b331cdf 100644 --- a/src/routes/auth/manage.ts +++ b/src/routes/auth/manage.ts @@ -32,7 +32,12 @@ export const manageAuthRouter = makeRouter((app) => { app.post( '/auth/register/start', { schema: { body: startSchema } }, - handle(async ({ em, body }) => { + handle(async ({ em, body, limiter, req }) => { + await limiter?.assertAndBump(req, { + id: 'register_challenge_tokens', + max: 10, + window: '10m', + }); await assertCaptcha(body.captchaToken); const challenge = new ChallengeCode(); @@ -50,7 +55,13 @@ export const manageAuthRouter = makeRouter((app) => { app.post( '/auth/register/complete', { schema: { body: completeSchema } }, - handle(async ({ em, body, req }) => { + handle(async ({ em, body, req, limiter }) => { + await limiter?.assertAndBump(req, { + id: 'register_complete', + max: 10, + window: '10m', + }); + await assertChallengeCode( em, body.challenge.code, diff --git a/src/routes/metrics.ts b/src/routes/metrics.ts index 074e364..b53863d 100644 --- a/src/routes/metrics.ts +++ b/src/routes/metrics.ts @@ -29,7 +29,14 @@ export const metricsRouter = makeRouter((app) => { body: metricsProviderInputSchema, }, }, - handle(async ({ em, body }) => { + handle(async ({ em, body, req, limiter }) => { + await limiter?.assertAndBump(req, { + id: 'provider_metrics', + max: 300, + inc: body.items.length, + window: '30m', + }); + const entities = body.items.map((v) => { const metric = new ProviderMetric(); em.assign(metric, { diff --git a/src/services/handler.ts b/src/services/handler.ts index d944221..f592993 100644 --- a/src/services/handler.ts +++ b/src/services/handler.ts @@ -1,4 +1,6 @@ import { getORM } from '@/modules/mikro'; +import { getLimiter } from '@/modules/ratelimits'; +import { Limiter } from '@/modules/ratelimits/limiter'; import { makeAuthContext } from '@/services/auth'; import { EntityManager } from '@mikro-orm/postgresql'; import { @@ -74,6 +76,7 @@ export type RequestContext< Logger >['query']; em: EntityManager; + limiter: Limiter | null; auth: ReturnType; }; @@ -124,6 +127,7 @@ export function handle< query: req.query, em, auth: makeAuthContext(em, req), + limiter: getLimiter(), }), ); };