From e5d91a9b9cc27b8de55dcf299c8569c89e23debb Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 24 Dec 2021 14:49:03 +0100 Subject: [PATCH] Upgrade redis dep --- package.json | 3 +- server/lib/plugins/plugin-manager.ts | 8 +- server/lib/redis.ts | 172 ++++--------------- server/middlewares/cache/shared/api-cache.ts | 50 +++--- yarn.lock | 66 +++++-- 5 files changed, 120 insertions(+), 179 deletions(-) diff --git a/package.json b/package.json index f5f2de33d..09081a54e 100644 --- a/package.json +++ b/package.json @@ -131,7 +131,7 @@ "prompt": "^1.0.0", "proxy-addr": "^2.0.7", "pug": "^3.0.0", - "redis": "^3.0.2", + "redis": "^4.0.1", "reflect-metadata": "^0.1.12", "sanitize-html": "2.x", "sequelize": "6.9.0", @@ -182,7 +182,6 @@ "@types/nodemailer": "^6.2.0", "@types/oauth2-server": "^3.0.8", "@types/pem": "^1.9.3", - "@types/redis": "^2.8.5", "@types/request": "^2.0.3", "@types/supertest": "^2.0.3", "@types/validator": "^13.0.0", diff --git a/server/lib/plugins/plugin-manager.ts b/server/lib/plugins/plugin-manager.ts index ff00ab9e8..39e7f9a5b 100644 --- a/server/lib/plugins/plugin-manager.ts +++ b/server/lib/plugins/plugin-manager.ts @@ -5,7 +5,13 @@ import { basename, join } from 'path' import { decachePlugin } from '@server/helpers/decache' import { MOAuthTokenUser, MUser } from '@server/types/models' import { getCompleteLocale } from '@shared/core-utils' -import { ClientScriptJSON, PluginPackageJSON, PluginTranslation, PluginTranslationPathsJSON, RegisterServerHookOptions } from '@shared/models' +import { + ClientScriptJSON, + PluginPackageJSON, + PluginTranslation, + PluginTranslationPathsJSON, + RegisterServerHookOptions +} from '@shared/models' import { getHookType, internalRunHook } from '../../../shared/core-utils/plugins/hooks' import { PluginType } from '../../../shared/models/plugins/plugin.type' import { ServerHook, ServerHookName } from '../../../shared/models/plugins/server/server-hook.model' diff --git a/server/lib/redis.ts b/server/lib/redis.ts index 8aec4b793..0478bfc89 100644 --- a/server/lib/redis.ts +++ b/server/lib/redis.ts @@ -1,31 +1,29 @@ import express from 'express' -import { createClient, RedisClient } from 'redis' +import { createClient } from 'redis' +import { exists } from '@server/helpers/custom-validators/misc' import { logger } from '../helpers/logger' import { generateRandomString } from '../helpers/utils' +import { CONFIG } from '../initializers/config' import { CONTACT_FORM_LIFETIME, - USER_EMAIL_VERIFY_LIFETIME, - USER_PASSWORD_RESET_LIFETIME, - USER_PASSWORD_CREATE_LIFETIME, - VIEW_LIFETIME, - WEBSERVER, + RESUMABLE_UPLOAD_SESSION_LIFETIME, TRACKER_RATE_LIMITS, - RESUMABLE_UPLOAD_SESSION_LIFETIME + USER_EMAIL_VERIFY_LIFETIME, + USER_PASSWORD_CREATE_LIFETIME, + USER_PASSWORD_RESET_LIFETIME, + VIEW_LIFETIME, + WEBSERVER } from '../initializers/constants' -import { CONFIG } from '../initializers/config' -import { exists } from '@server/helpers/custom-validators/misc' -type CachedRoute = { - body: string - contentType?: string - statusCode?: string -} +// Only used for typings +const redisClientWrapperForType = () => createClient<{}>() class Redis { private static instance: Redis private initialized = false - private client: RedisClient + private connected = false + private client: ReturnType private prefix: string private constructor () { @@ -38,21 +36,24 @@ class Redis { this.client = createClient(Redis.getRedisClientOptions()) + this.client.connect() + .then(() => { this.connected = true }) + .catch(err => { + logger.error('Cannot connect to redis', { err }) + process.exit(-1) + }) + this.client.on('error', err => { logger.error('Error in Redis client.', { err }) process.exit(-1) }) - if (CONFIG.REDIS.AUTH) { - this.client.auth(CONFIG.REDIS.AUTH) - } - this.prefix = 'redis-' + WEBSERVER.HOST + '-' } static getRedisClientOptions () { return Object.assign({}, - (CONFIG.REDIS.AUTH && CONFIG.REDIS.AUTH != null) ? { password: CONFIG.REDIS.AUTH } : {}, + CONFIG.REDIS.AUTH ? { password: CONFIG.REDIS.AUTH } : {}, (CONFIG.REDIS.DB) ? { db: CONFIG.REDIS.DB } : {}, (CONFIG.REDIS.HOSTNAME && CONFIG.REDIS.PORT) ? { host: CONFIG.REDIS.HOSTNAME, port: CONFIG.REDIS.PORT } @@ -68,6 +69,10 @@ class Redis { return this.prefix } + isConnected () { + return this.connected + } + /* ************ Forgot password ************ */ async setResetPasswordVerificationString (userId: number) { @@ -146,25 +151,6 @@ class Redis { return this.exists(this.generateTrackerBlockIPKey(ip)) } - /* ************ API cache ************ */ - - async getCachedRoute (req: express.Request) { - const cached = await this.getObject(this.generateCachedRouteKey(req)) - - return cached as CachedRoute - } - - setCachedRoute (req: express.Request, body: any, lifetime: number, contentType?: string, statusCode?: number) { - const cached: CachedRoute = Object.assign( - {}, - { body: body.toString() }, - (contentType) ? { contentType } : null, - (statusCode) ? { statusCode: statusCode.toString() } : null - ) - - return this.setObject(this.generateCachedRouteKey(req), cached, lifetime) - } - /* ************ Video views stats ************ */ addVideoViewStats (videoId: number) { @@ -277,10 +263,6 @@ class Redis { /* ************ Keys generation ************ */ - generateCachedRouteKey (req: express.Request) { - return req.method + '-' + req.originalUrl - } - private generateLocalVideoViewsKeys (videoId?: Number) { return { setKey: `local-video-views-buffer`, videoKey: `local-video-views-buffer-${videoId}` } } @@ -320,125 +302,45 @@ class Redis { /* ************ Redis helpers ************ */ private getValue (key: string) { - return new Promise((res, rej) => { - this.client.get(this.prefix + key, (err, value) => { - if (err) return rej(err) - - return res(value) - }) - }) + return this.client.get(this.prefix + key) } private getSet (key: string) { - return new Promise((res, rej) => { - this.client.smembers(this.prefix + key, (err, value) => { - if (err) return rej(err) - - return res(value) - }) - }) + return this.client.sMembers(this.prefix + key) } private addToSet (key: string, value: string) { - return new Promise((res, rej) => { - this.client.sadd(this.prefix + key, value, err => err ? rej(err) : res()) - }) + return this.client.sAdd(this.prefix + key, value) } private deleteFromSet (key: string, value: string) { - return new Promise((res, rej) => { - this.client.srem(this.prefix + key, value, err => err ? rej(err) : res()) - }) + return this.client.sRem(this.prefix + key, value) } private deleteKey (key: string) { - return new Promise((res, rej) => { - this.client.del(this.prefix + key, err => err ? rej(err) : res()) - }) + return this.client.del(this.prefix + key) } - private deleteFieldInHash (key: string, field: string) { - return new Promise((res, rej) => { - this.client.hdel(this.prefix + key, field, err => err ? rej(err) : res()) - }) - } + private async setValue (key: string, value: string, expirationMilliseconds: number) { + const result = await this.client.set(this.prefix + key, value, { PX: expirationMilliseconds }) - private setValue (key: string, value: string, expirationMilliseconds: number) { - return new Promise((res, rej) => { - this.client.set(this.prefix + key, value, 'PX', expirationMilliseconds, (err, ok) => { - if (err) return rej(err) - - if (ok !== 'OK') return rej(new Error('Redis set result is not OK.')) - - return res() - }) - }) + if (result !== 'OK') throw new Error('Redis set result is not OK.') } private removeValue (key: string) { - return new Promise((res, rej) => { - this.client.del(this.prefix + key, err => { - if (err) return rej(err) - - return res() - }) - }) - } - - private setObject (key: string, obj: { [id: string]: string }, expirationMilliseconds: number) { - return new Promise((res, rej) => { - this.client.hmset(this.prefix + key, obj, (err, ok) => { - if (err) return rej(err) - if (!ok) return rej(new Error('Redis mset result is not OK.')) - - this.client.pexpire(this.prefix + key, expirationMilliseconds, (err, ok) => { - if (err) return rej(err) - if (!ok) return rej(new Error('Redis expiration result is not OK.')) - - return res() - }) - }) - }) + return this.client.del(this.prefix + key) } private getObject (key: string) { - return new Promise<{ [id: string]: string }>((res, rej) => { - this.client.hgetall(this.prefix + key, (err, value) => { - if (err) return rej(err) - - return res(value) - }) - }) - } - - private setValueInHash (key: string, field: string, value: string) { - return new Promise((res, rej) => { - this.client.hset(this.prefix + key, field, value, (err) => { - if (err) return rej(err) - - return res() - }) - }) + return this.client.hGetAll(this.prefix + key) } private increment (key: string) { - return new Promise((res, rej) => { - this.client.incr(this.prefix + key, (err, value) => { - if (err) return rej(err) - - return res(value) - }) - }) + return this.client.incr(this.prefix + key) } private exists (key: string) { - return new Promise((res, rej) => { - this.client.exists(this.prefix + key, (err, existsNumber) => { - if (err) return rej(err) - - return res(existsNumber === 1) - }) - }) + return this.client.exists(this.prefix + key) } static get Instance () { diff --git a/server/middlewares/cache/shared/api-cache.ts b/server/middlewares/cache/shared/api-cache.ts index f8846dcfc..86c5095b5 100644 --- a/server/middlewares/cache/shared/api-cache.ts +++ b/server/middlewares/cache/shared/api-cache.ts @@ -7,6 +7,7 @@ import { isTestInstance, parseDurationToMs } from '@server/helpers/core-utils' import { logger } from '@server/helpers/logger' import { Redis } from '@server/lib/redis' import { HttpStatusCode } from '@shared/models' +import { asyncMiddleware } from '@server/middlewares' export interface APICacheOptions { headerBlacklist?: string[] @@ -40,24 +41,25 @@ export class ApiCache { buildMiddleware (strDuration: string) { const duration = parseDurationToMs(strDuration) - return (req: express.Request, res: express.Response, next: express.NextFunction) => { - const key = Redis.Instance.getPrefix() + 'api-cache-' + req.originalUrl - const redis = Redis.Instance.getClient() + return asyncMiddleware( + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + const key = Redis.Instance.getPrefix() + 'api-cache-' + req.originalUrl + const redis = Redis.Instance.getClient() - if (!redis.connected) return this.makeResponseCacheable(res, next, key, duration) + if (!Redis.Instance.isConnected()) return this.makeResponseCacheable(res, next, key, duration) - try { - redis.hgetall(key, (err, obj) => { - if (!err && obj && obj.response) { + try { + const obj = await redis.hGetAll(key) + if (obj?.response) { return this.sendCachedResponse(req, res, JSON.parse(obj.response), duration) } return this.makeResponseCacheable(res, next, key, duration) - }) - } catch (err) { - return this.makeResponseCacheable(res, next, key, duration) + } catch (err) { + return this.makeResponseCacheable(res, next, key, duration) + } } - } + ) } private shouldCacheResponse (response: express.Response) { @@ -93,21 +95,22 @@ export class ApiCache { } as CacheObject } - private cacheResponse (key: string, value: object, duration: number) { + private async cacheResponse (key: string, value: object, duration: number) { const redis = Redis.Instance.getClient() - if (redis.connected) { - try { - redis.hset(key, 'response', JSON.stringify(value)) - redis.hset(key, 'duration', duration + '') + if (Redis.Instance.isConnected()) { + await Promise.all([ + redis.hSet(key, 'response', JSON.stringify(value)), + redis.hSet(key, 'duration', duration + ''), redis.expire(key, duration / 1000) - } catch (err) { - logger.error('Cannot set cache in redis.', { err }) - } + ]) } // add automatic cache clearing from duration, includes max limit on setTimeout - this.timers[key] = setTimeout(() => this.clear(key), Math.min(duration, 2147483647)) + this.timers[key] = setTimeout(() => { + this.clear(key) + .catch(err => logger.error('Cannot clear Redis key %s.', key, { err })) + }, Math.min(duration, 2147483647)) } private accumulateContent (res: express.Response, content: any) { @@ -184,6 +187,7 @@ export class ApiCache { encoding ) self.cacheResponse(key, cacheObject, duration) + .catch(err => logger.error('Cannot cache response', { err })) } } @@ -235,7 +239,7 @@ export class ApiCache { return response.end(data, cacheObject.encoding) } - private clear (target: string) { + private async clear (target: string) { const redis = Redis.Instance.getClient() if (target) { @@ -243,7 +247,7 @@ export class ApiCache { delete this.timers[target] try { - redis.del(target) + await redis.del(target) } catch (err) { logger.error('Cannot delete %s in redis cache.', target, { err }) } @@ -255,7 +259,7 @@ export class ApiCache { delete this.timers[key] try { - redis.del(key) + await redis.del(key) } catch (err) { logger.error('Cannot delete %s in redis cache.', key, { err }) } diff --git a/yarn.lock b/yarn.lock index f7d41f2ac..5320b106b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1365,6 +1365,31 @@ semver "^7.3.5" tar "^6.1.11" +"@node-redis/client@^1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@node-redis/client/-/client-1.0.1.tgz#ddca6021097ce1026fedc193cac8c36b05c6cad8" + integrity sha512-o0I4LdzJXP6QYxRnBPrQ7cIG5tF3SNM/PBnjC3mV6QkzIiGRElzWqSr9a9JCJdcyB1SIA80bhgGhpdTpCQ1Sdw== + dependencies: + cluster-key-slot "1.1.0" + generic-pool "3.8.2" + redis-parser "3.0.0" + yallist "4.0.0" + +"@node-redis/json@^1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@node-redis/json/-/json-1.0.1.tgz#8cd987c1855392adf21bc4f06163a7eda97a40a3" + integrity sha512-2EB96ZN0yUr4mgA9Odme48jX8eF5Ji0jrsRn4rLfEhME7L3rHLdKeUfxJKxbPOxadP6k8+6ViElxPZrKuV2nvQ== + +"@node-redis/search@^1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@node-redis/search/-/search-1.0.1.tgz#8d0936049f4858b9aefab40524ce8e5a52e5d08e" + integrity sha512-iA2Gw6gr0X6IfNSjTyme9W1tDlLkwQ1bPApo4s8aVwZ2Ju8Z4COVik0vT6BJPRin79f5xPZgnaec3DIoC2UpHA== + +"@node-redis/time-series@^1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@node-redis/time-series/-/time-series-1.0.0.tgz#3db4caa63d7c158f0b73ab6cd46bd6c9c187dfaf" + integrity sha512-QcaCIL/DlYJXedSfmjF+IRxKJbBUXBrjA5Gv0IiPlXXFFOkRnbPGKq6hmwBAAWyk1U03wyBHDFKVS3/9GnZV8g== + "@nodelib/fs.scandir@2.1.5": version "2.1.5" resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5" @@ -1805,7 +1830,7 @@ resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc" integrity sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw== -"@types/redis@^2.8.0", "@types/redis@^2.8.5": +"@types/redis@^2.8.0": version "2.8.32" resolved "https://registry.yarnpkg.com/@types/redis/-/redis-2.8.32.tgz#1d3430219afbee10f8cfa389dad2571a05ecfb11" integrity sha512-7jkMKxcGq9p242exlbsVzuJb57KqHRhNl4dHoQu2Y5v9bCAbtIXXH0R3HleSQW4CTOqpHIYUW3t6tpUj4BVQ+w== @@ -2995,7 +3020,7 @@ clone@^2.0.0: resolved "https://registry.yarnpkg.com/clone/-/clone-2.1.2.tgz#1b7f4b9f591f1e8f83670401600345a02887435f" integrity sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18= -cluster-key-slot@^1.1.0: +cluster-key-slot@1.1.0, cluster-key-slot@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d" integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw== @@ -3475,7 +3500,7 @@ delegates@^1.0.0: resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" integrity sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o= -denque@^1.1.0, denque@^1.5.0: +denque@^1.1.0: version "1.5.1" resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.1.tgz#07f670e29c9a78f8faecb2566a1e2c11929c5cbf" integrity sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw== @@ -4479,6 +4504,11 @@ gauge@^3.0.0: strip-ansi "^6.0.1" wide-align "^1.1.2" +generic-pool@3.8.2: + version "3.8.2" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.8.2.tgz#aab4f280adb522fdfbdc5e5b64d718d3683f04e9" + integrity sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg== + get-browser-rtc@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/get-browser-rtc/-/get-browser-rtc-1.1.0.tgz#d1494e299b00f33fc8e9d6d3343ba4ba99711a2c" @@ -7352,7 +7382,7 @@ record-cache@^1.0.2: resolved "https://registry.yarnpkg.com/record-cache/-/record-cache-1.1.1.tgz#ba3088a489f50491a4af7b14d410822c394fb811" integrity sha512-L5hZlgWc7CmGbztnemQoKE1bLu9rtI2skOB0ttE4C5+TVszLE8Rd0YLTROSgvXKLAqPumS/soyN5tJW5wJLmJQ== -redis-commands@1.7.0, redis-commands@^1.7.0: +redis-commands@1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.7.0.tgz#15a6fea2d58281e27b1cd1acfb4b293e278c3a89" integrity sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ== @@ -7362,22 +7392,22 @@ redis-errors@^1.0.0, redis-errors@^1.2.0: resolved "https://registry.yarnpkg.com/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad" integrity sha1-62LSrbFeTq9GEMBK/hUpOEJQq60= -redis-parser@^3.0.0: +redis-parser@3.0.0, redis-parser@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-3.0.0.tgz#b66d828cdcafe6b4b8a428a7def4c6bcac31c8b4" integrity sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ= dependencies: redis-errors "^1.0.0" -redis@^3.0.2: - version "3.1.2" - resolved "https://registry.yarnpkg.com/redis/-/redis-3.1.2.tgz#766851117e80653d23e0ed536254677ab647638c" - integrity sha512-grn5KoZLr/qrRQVwoSkmzdbw6pwF+/rwODtrOr6vuBRiR/f3rjSTGupbF90Zpqm2oenix8Do6RV7pYEkGwlKkw== +redis@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/redis/-/redis-4.0.1.tgz#c020e2ac7f83f0c1d42ced50b8a7af28164bd6ee" + integrity sha512-qfcq1oz2ci7pNdCfTLLEuKhS8jZ17dFiT1exogOr+jd3EVP/h9qpy7K+VajB4BXA0k8q68KFqR6HrliKV6jt1Q== dependencies: - denque "^1.5.0" - redis-commands "^1.7.0" - redis-errors "^1.2.0" - redis-parser "^3.0.0" + "@node-redis/client" "^1.0.1" + "@node-redis/json" "^1.0.1" + "@node-redis/search" "^1.0.1" + "@node-redis/time-series" "^1.0.0" reflect-metadata@^0.1.12: version "0.1.13" @@ -9050,16 +9080,16 @@ y18n@^5.0.5: resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== +yallist@4.0.0, yallist@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" + integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== + yallist@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52" integrity sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI= -yallist@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" - integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== - yaml@^1.10.0: version "1.10.2" resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b"