From 7348b1fd84dee869b3c36554aea6797f09d4ceed Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 14 Sep 2018 11:52:23 +0200 Subject: [PATCH] Speed up overviews route --- package.json | 2 + server/controllers/api/overviews.ts | 26 +++++++++---- server/helpers/utils.ts | 24 +++--------- server/initializers/constants.ts | 5 +++ .../schedulers/videos-redundancy-scheduler.ts | 2 +- server/models/video/video.ts | 27 ++++++------- yarn.lock | 38 +++++++++++++++++-- 7 files changed, 80 insertions(+), 44 deletions(-) diff --git a/package.json b/package.json index 5a8843b0c..1cb5be181 100644 --- a/package.json +++ b/package.json @@ -111,6 +111,7 @@ "jsonld-signatures": "https://github.com/Chocobozzz/jsonld-signatures#rsa2017", "lodash": "^4.17.10", "magnet-uri": "^5.1.4", + "memoizee": "^0.4.14", "morgan": "^1.5.3", "multer": "^1.1.0", "nodemailer": "^4.4.2", @@ -158,6 +159,7 @@ "@types/lodash": "^4.14.64", "@types/magnet-uri": "^5.1.1", "@types/maildev": "^0.0.1", + "@types/memoizee": "^0.4.2", "@types/mkdirp": "^0.5.1", "@types/mocha": "^5.0.0", "@types/morgan": "^1.7.32", diff --git a/server/controllers/api/overviews.ts b/server/controllers/api/overviews.ts index da941c0ac..cc3cc54a7 100644 --- a/server/controllers/api/overviews.ts +++ b/server/controllers/api/overviews.ts @@ -4,8 +4,9 @@ import { VideoModel } from '../../models/video/video' import { asyncMiddleware } from '../../middlewares' import { TagModel } from '../../models/video/tag' import { VideosOverview } from '../../../shared/models/overviews' -import { OVERVIEWS, ROUTE_CACHE_LIFETIME } from '../../initializers' +import { MEMOIZE_TTL, OVERVIEWS, ROUTE_CACHE_LIFETIME } from '../../initializers' import { cacheRoute } from '../../middlewares/cache' +import * as memoizee from 'memoizee' const overviewsRouter = express.Router() @@ -23,10 +24,17 @@ export { overviewsRouter } // This endpoint could be quite long, but we cache it async function getVideosOverview (req: express.Request, res: express.Response) { const attributes = await buildSamples() + + const [ categories, channels, tags ] = await Promise.all([ + Promise.all(attributes.categories.map(c => getVideosByCategory(c, res))), + Promise.all(attributes.channels.map(c => getVideosByChannel(c, res))), + Promise.all(attributes.tags.map(t => getVideosByTag(t, res))) + ]) + const result: VideosOverview = { - categories: await Promise.all(attributes.categories.map(c => getVideosByCategory(c, res))), - channels: await Promise.all(attributes.channels.map(c => getVideosByChannel(c, res))), - tags: await Promise.all(attributes.tags.map(t => getVideosByTag(t, res))) + categories, + channels, + tags } // Cleanup our object @@ -37,7 +45,7 @@ async function getVideosOverview (req: express.Request, res: express.Response) { return res.json(result) } -async function buildSamples () { +const buildSamples = memoizee(async function () { const [ categories, channels, tags ] = await Promise.all([ VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT), VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT), @@ -45,7 +53,7 @@ async function buildSamples () { ]) return { categories, channels, tags } -} +}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE }) async function getVideosByTag (tag: string, res: express.Response) { const videos = await getVideos(res, { tagsOneOf: [ tag ] }) @@ -84,14 +92,16 @@ async function getVideos ( res: express.Response, where: { videoChannelId?: number, tagsOneOf?: string[], categoryOneOf?: number[] } ) { - const { data } = await VideoModel.listForApi(Object.assign({ + const query = Object.assign({ start: 0, count: 10, sort: '-createdAt', includeLocalVideos: true, nsfw: buildNSFWFilter(res), withFiles: false - }, where)) + }, where) + + const { data } = await VideoModel.listForApi(query, false) return data.map(d => d.toFormattedJSON()) } diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index a1ed8e72d..a42474417 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts @@ -1,12 +1,12 @@ import { ResultList } from '../../shared' import { CONFIG } from '../initializers' -import { ActorModel } from '../models/activitypub/actor' import { ApplicationModel } from '../models/application/application' import { pseudoRandomBytesPromise, sha256 } from './core-utils' import { logger } from './logger' import { join } from 'path' import { Instance as ParseTorrent } from 'parse-torrent' import { remove } from 'fs-extra' +import * as memoizee from 'memoizee' function deleteFileAsync (path: string) { remove(path) @@ -36,24 +36,12 @@ function getFormattedObjects (objects: T[], obje } as ResultList } -async function getServerActor () { - if (getServerActor.serverActor === undefined) { - const application = await ApplicationModel.load() - if (!application) throw Error('Could not load Application from database.') +const getServerActor = memoizee(async function () { + const application = await ApplicationModel.load() + if (!application) throw Error('Could not load Application from database.') - getServerActor.serverActor = application.Account.Actor - } - - if (!getServerActor.serverActor) { - logger.error('Cannot load server actor.') - process.exit(0) - } - - return Promise.resolve(getServerActor.serverActor) -} -namespace getServerActor { - export let serverActor: ActorModel -} + return application.Account.Actor +}) function generateVideoTmpPath (target: string | ParseTorrent) { const id = typeof target === 'string' ? target : target.infoHash diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 5f7bcbd48..9cccb0919 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -592,6 +592,10 @@ const CACHE = { } } +const MEMOIZE_TTL = { + OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours +} + const REDUNDANCY = { VIDEOS: { EXPIRES_AFTER_MS: 48 * 3600 * 1000, // 2 days @@ -708,6 +712,7 @@ export { VIDEO_ABUSE_STATES, JOB_REQUEST_TIMEOUT, USER_PASSWORD_RESET_LIFETIME, + MEMOIZE_TTL, USER_EMAIL_VERIFY_LIFETIME, IMAGE_MIMETYPE_EXT, OVERVIEWS, diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 8b91d750b..7079600a9 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -81,7 +81,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } if (cache.strategy === 'recently-added') { - const minViews = (cache as RecentlyAddedStrategy).minViews + const minViews = cache.minViews return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews) } } diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 23d1dedd6..b7d3f184f 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -929,7 +929,7 @@ export class VideoModel extends Model { videoChannelId?: number, actorId?: number trendingDays?: number - }) { + }, countVideos = true) { const query: IFindOptions = { offset: options.start, limit: options.count, @@ -962,7 +962,7 @@ export class VideoModel extends Model { trendingDays } - return VideoModel.getAvailableForApi(query, queryOptions) + return VideoModel.getAvailableForApi(query, queryOptions, countVideos) } static async searchAndPopulateAccountAndServer (options: { @@ -1164,7 +1164,14 @@ export class VideoModel extends Model { } // threshold corresponds to how many video the field should have to be returned - static getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) { + static async getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) { + const actorId = (await getServerActor()).id + + const scopeOptions = { + actorId, + includeLocalVideos: true + } + const query: IFindOptions = { attributes: [ field ], limit: count, @@ -1172,17 +1179,11 @@ export class VideoModel extends Model { having: Sequelize.where(Sequelize.fn('COUNT', Sequelize.col(field)), { [ Sequelize.Op.gte ]: threshold }) as any, // FIXME: typings - where: { - [ field ]: { - [ Sequelize.Op.not ]: null - }, - privacy: VideoPrivacy.PUBLIC, - state: VideoState.PUBLISHED - }, order: [ this.sequelize.random() ] } - return VideoModel.findAll(query) + return VideoModel.scope({ method: [ ScopeNames.AVAILABLE_FOR_LIST_IDS, scopeOptions ] }) + .findAll(query) .then(rows => rows.map(r => r[ field ])) } @@ -1210,7 +1211,7 @@ export class VideoModel extends Model { return {} } - private static async getAvailableForApi (query: IFindOptions, options: AvailableForListIDsOptions) { + private static async getAvailableForApi (query: IFindOptions, options: AvailableForListIDsOptions, countVideos = true) { const idsScope = { method: [ ScopeNames.AVAILABLE_FOR_LIST_IDS, options @@ -1227,7 +1228,7 @@ export class VideoModel extends Model { } const [ count, rowsId ] = await Promise.all([ - VideoModel.scope(countScope).count(countQuery), + countVideos ? VideoModel.scope(countScope).count(countQuery) : Promise.resolve(undefined), VideoModel.scope(idsScope).findAll(query) ]) const ids = rowsId.map(r => r.id) diff --git a/yarn.lock b/yarn.lock index c8fb21117..52ff895b1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -160,6 +160,10 @@ dependencies: "@types/node" "*" +"@types/memoizee@^0.4.2": + version "0.4.2" + resolved "https://registry.yarnpkg.com/@types/memoizee/-/memoizee-0.4.2.tgz#a500158999a8144a9b46cf9a9fb49b15f1853573" + "@types/mime@*": version "2.0.0" resolved "https://registry.yarnpkg.com/@types/mime/-/mime-2.0.0.tgz#5a7306e367c539b9f6543499de8dd519fac37a8b" @@ -2058,7 +2062,7 @@ error@^7.0.0: string-template "~0.2.1" xtend "~4.0.0" -es5-ext@^0.10.14, es5-ext@^0.10.35, es5-ext@^0.10.9, es5-ext@~0.10.14: +es5-ext@^0.10.14, es5-ext@^0.10.35, es5-ext@^0.10.45, es5-ext@^0.10.9, es5-ext@~0.10.14, es5-ext@~0.10.2: version "0.10.46" resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.46.tgz#efd99f67c5a7ec789baa3daa7f79870388f7f572" dependencies: @@ -2110,7 +2114,7 @@ es6-symbol@3.1.1, es6-symbol@^3.1.1, es6-symbol@~3.1.1: d "1" es5-ext "~0.10.14" -es6-weak-map@^2.0.1: +es6-weak-map@^2.0.1, es6-weak-map@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/es6-weak-map/-/es6-weak-map-2.0.2.tgz#5e3ab32251ffd1538a1f8e5ffa1357772f92d96f" dependencies: @@ -2223,7 +2227,7 @@ etag@~1.8.1: version "1.8.1" resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" -event-emitter@~0.3.5: +event-emitter@^0.3.5, event-emitter@~0.3.5: version "0.3.5" resolved "https://registry.yarnpkg.com/event-emitter/-/event-emitter-0.3.5.tgz#df8c69eef1647923c7157b9ce83840610b02cc39" dependencies: @@ -3757,7 +3761,7 @@ is-plain-object@^2.0.1, is-plain-object@^2.0.3, is-plain-object@^2.0.4: dependencies: isobject "^3.0.1" -is-promise@^2.1.0: +is-promise@^2.1, is-promise@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/is-promise/-/is-promise-2.1.0.tgz#79a2a9ece7f096e80f36d2b2f3bc16c1ff4bf3fa" @@ -4490,6 +4494,12 @@ lru-cache@4.1.x, lru-cache@^4.0.1: pseudomap "^1.0.2" yallist "^2.1.2" +lru-queue@0.1: + version "0.1.0" + resolved "https://registry.yarnpkg.com/lru-queue/-/lru-queue-0.1.0.tgz#2738bd9f0d3cf4f84490c5736c48699ac632cda3" + dependencies: + es5-ext "~0.10.2" + lru@^3.0.0, lru@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/lru/-/lru-3.1.0.tgz#ea7fb8546d83733396a13091d76cfeb4c06837d5" @@ -4594,6 +4604,19 @@ mem@^1.1.0: dependencies: mimic-fn "^1.0.0" +memoizee@^0.4.14: + version "0.4.14" + resolved "https://registry.yarnpkg.com/memoizee/-/memoizee-0.4.14.tgz#07a00f204699f9a95c2d9e77218271c7cd610d57" + dependencies: + d "1" + es5-ext "^0.10.45" + es6-weak-map "^2.0.2" + event-emitter "^0.3.5" + is-promise "^2.1" + lru-queue "0.1" + next-tick "1" + timers-ext "^0.1.5" + memory-chunk-store@^1.2.0: version "1.3.0" resolved "https://registry.yarnpkg.com/memory-chunk-store/-/memory-chunk-store-1.3.0.tgz#ae99e7e3b58b52db43d49d94722930d39459d0c4" @@ -7201,6 +7224,13 @@ timed-out@^4.0.0: version "4.0.1" resolved "https://registry.yarnpkg.com/timed-out/-/timed-out-4.0.1.tgz#f32eacac5a175bea25d7fab565ab3ed8741ef56f" +timers-ext@^0.1.5: + version "0.1.5" + resolved "https://registry.yarnpkg.com/timers-ext/-/timers-ext-0.1.5.tgz#77147dd4e76b660c2abb8785db96574cbbd12922" + dependencies: + es5-ext "~0.10.14" + next-tick "1" + tiny-lr@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/tiny-lr/-/tiny-lr-1.1.1.tgz#9fa547412f238fedb068ee295af8b682c98b2aab"