mirror of
https://github.com/Chocobozzz/PeerTube.git
synced 2024-08-17 17:30:32 +03:00
Speed up overviews route
This commit is contained in:
parent
2b62cccd75
commit
7348b1fd84
@ -111,6 +111,7 @@
|
||||
"jsonld-signatures": "https://github.com/Chocobozzz/jsonld-signatures#rsa2017",
|
||||
"lodash": "^4.17.10",
|
||||
"magnet-uri": "^5.1.4",
|
||||
"memoizee": "^0.4.14",
|
||||
"morgan": "^1.5.3",
|
||||
"multer": "^1.1.0",
|
||||
"nodemailer": "^4.4.2",
|
||||
@ -158,6 +159,7 @@
|
||||
"@types/lodash": "^4.14.64",
|
||||
"@types/magnet-uri": "^5.1.1",
|
||||
"@types/maildev": "^0.0.1",
|
||||
"@types/memoizee": "^0.4.2",
|
||||
"@types/mkdirp": "^0.5.1",
|
||||
"@types/mocha": "^5.0.0",
|
||||
"@types/morgan": "^1.7.32",
|
||||
|
@ -4,8 +4,9 @@ import { VideoModel } from '../../models/video/video'
|
||||
import { asyncMiddleware } from '../../middlewares'
|
||||
import { TagModel } from '../../models/video/tag'
|
||||
import { VideosOverview } from '../../../shared/models/overviews'
|
||||
import { OVERVIEWS, ROUTE_CACHE_LIFETIME } from '../../initializers'
|
||||
import { MEMOIZE_TTL, OVERVIEWS, ROUTE_CACHE_LIFETIME } from '../../initializers'
|
||||
import { cacheRoute } from '../../middlewares/cache'
|
||||
import * as memoizee from 'memoizee'
|
||||
|
||||
const overviewsRouter = express.Router()
|
||||
|
||||
@ -23,10 +24,17 @@ export { overviewsRouter }
|
||||
// This endpoint could be quite long, but we cache it
|
||||
async function getVideosOverview (req: express.Request, res: express.Response) {
|
||||
const attributes = await buildSamples()
|
||||
|
||||
const [ categories, channels, tags ] = await Promise.all([
|
||||
Promise.all(attributes.categories.map(c => getVideosByCategory(c, res))),
|
||||
Promise.all(attributes.channels.map(c => getVideosByChannel(c, res))),
|
||||
Promise.all(attributes.tags.map(t => getVideosByTag(t, res)))
|
||||
])
|
||||
|
||||
const result: VideosOverview = {
|
||||
categories: await Promise.all(attributes.categories.map(c => getVideosByCategory(c, res))),
|
||||
channels: await Promise.all(attributes.channels.map(c => getVideosByChannel(c, res))),
|
||||
tags: await Promise.all(attributes.tags.map(t => getVideosByTag(t, res)))
|
||||
categories,
|
||||
channels,
|
||||
tags
|
||||
}
|
||||
|
||||
// Cleanup our object
|
||||
@ -37,7 +45,7 @@ async function getVideosOverview (req: express.Request, res: express.Response) {
|
||||
return res.json(result)
|
||||
}
|
||||
|
||||
async function buildSamples () {
|
||||
const buildSamples = memoizee(async function () {
|
||||
const [ categories, channels, tags ] = await Promise.all([
|
||||
VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT),
|
||||
VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT),
|
||||
@ -45,7 +53,7 @@ async function buildSamples () {
|
||||
])
|
||||
|
||||
return { categories, channels, tags }
|
||||
}
|
||||
}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE })
|
||||
|
||||
async function getVideosByTag (tag: string, res: express.Response) {
|
||||
const videos = await getVideos(res, { tagsOneOf: [ tag ] })
|
||||
@ -84,14 +92,16 @@ async function getVideos (
|
||||
res: express.Response,
|
||||
where: { videoChannelId?: number, tagsOneOf?: string[], categoryOneOf?: number[] }
|
||||
) {
|
||||
const { data } = await VideoModel.listForApi(Object.assign({
|
||||
const query = Object.assign({
|
||||
start: 0,
|
||||
count: 10,
|
||||
sort: '-createdAt',
|
||||
includeLocalVideos: true,
|
||||
nsfw: buildNSFWFilter(res),
|
||||
withFiles: false
|
||||
}, where))
|
||||
}, where)
|
||||
|
||||
const { data } = await VideoModel.listForApi(query, false)
|
||||
|
||||
return data.map(d => d.toFormattedJSON())
|
||||
}
|
||||
|
@ -1,12 +1,12 @@
|
||||
import { ResultList } from '../../shared'
|
||||
import { CONFIG } from '../initializers'
|
||||
import { ActorModel } from '../models/activitypub/actor'
|
||||
import { ApplicationModel } from '../models/application/application'
|
||||
import { pseudoRandomBytesPromise, sha256 } from './core-utils'
|
||||
import { logger } from './logger'
|
||||
import { join } from 'path'
|
||||
import { Instance as ParseTorrent } from 'parse-torrent'
|
||||
import { remove } from 'fs-extra'
|
||||
import * as memoizee from 'memoizee'
|
||||
|
||||
function deleteFileAsync (path: string) {
|
||||
remove(path)
|
||||
@ -36,24 +36,12 @@ function getFormattedObjects<U, T extends FormattableToJSON> (objects: T[], obje
|
||||
} as ResultList<U>
|
||||
}
|
||||
|
||||
async function getServerActor () {
|
||||
if (getServerActor.serverActor === undefined) {
|
||||
const application = await ApplicationModel.load()
|
||||
if (!application) throw Error('Could not load Application from database.')
|
||||
const getServerActor = memoizee(async function () {
|
||||
const application = await ApplicationModel.load()
|
||||
if (!application) throw Error('Could not load Application from database.')
|
||||
|
||||
getServerActor.serverActor = application.Account.Actor
|
||||
}
|
||||
|
||||
if (!getServerActor.serverActor) {
|
||||
logger.error('Cannot load server actor.')
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
return Promise.resolve(getServerActor.serverActor)
|
||||
}
|
||||
namespace getServerActor {
|
||||
export let serverActor: ActorModel
|
||||
}
|
||||
return application.Account.Actor
|
||||
})
|
||||
|
||||
function generateVideoTmpPath (target: string | ParseTorrent) {
|
||||
const id = typeof target === 'string' ? target : target.infoHash
|
||||
|
@ -592,6 +592,10 @@ const CACHE = {
|
||||
}
|
||||
}
|
||||
|
||||
const MEMOIZE_TTL = {
|
||||
OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours
|
||||
}
|
||||
|
||||
const REDUNDANCY = {
|
||||
VIDEOS: {
|
||||
EXPIRES_AFTER_MS: 48 * 3600 * 1000, // 2 days
|
||||
@ -708,6 +712,7 @@ export {
|
||||
VIDEO_ABUSE_STATES,
|
||||
JOB_REQUEST_TIMEOUT,
|
||||
USER_PASSWORD_RESET_LIFETIME,
|
||||
MEMOIZE_TTL,
|
||||
USER_EMAIL_VERIFY_LIFETIME,
|
||||
IMAGE_MIMETYPE_EXT,
|
||||
OVERVIEWS,
|
||||
|
@ -81,7 +81,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
|
||||
}
|
||||
|
||||
if (cache.strategy === 'recently-added') {
|
||||
const minViews = (cache as RecentlyAddedStrategy).minViews
|
||||
const minViews = cache.minViews
|
||||
return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews)
|
||||
}
|
||||
}
|
||||
|
@ -929,7 +929,7 @@ export class VideoModel extends Model<VideoModel> {
|
||||
videoChannelId?: number,
|
||||
actorId?: number
|
||||
trendingDays?: number
|
||||
}) {
|
||||
}, countVideos = true) {
|
||||
const query: IFindOptions<VideoModel> = {
|
||||
offset: options.start,
|
||||
limit: options.count,
|
||||
@ -962,7 +962,7 @@ export class VideoModel extends Model<VideoModel> {
|
||||
trendingDays
|
||||
}
|
||||
|
||||
return VideoModel.getAvailableForApi(query, queryOptions)
|
||||
return VideoModel.getAvailableForApi(query, queryOptions, countVideos)
|
||||
}
|
||||
|
||||
static async searchAndPopulateAccountAndServer (options: {
|
||||
@ -1164,7 +1164,14 @@ export class VideoModel extends Model<VideoModel> {
|
||||
}
|
||||
|
||||
// threshold corresponds to how many video the field should have to be returned
|
||||
static getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) {
|
||||
static async getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) {
|
||||
const actorId = (await getServerActor()).id
|
||||
|
||||
const scopeOptions = {
|
||||
actorId,
|
||||
includeLocalVideos: true
|
||||
}
|
||||
|
||||
const query: IFindOptions<VideoModel> = {
|
||||
attributes: [ field ],
|
||||
limit: count,
|
||||
@ -1172,17 +1179,11 @@ export class VideoModel extends Model<VideoModel> {
|
||||
having: Sequelize.where(Sequelize.fn('COUNT', Sequelize.col(field)), {
|
||||
[ Sequelize.Op.gte ]: threshold
|
||||
}) as any, // FIXME: typings
|
||||
where: {
|
||||
[ field ]: {
|
||||
[ Sequelize.Op.not ]: null
|
||||
},
|
||||
privacy: VideoPrivacy.PUBLIC,
|
||||
state: VideoState.PUBLISHED
|
||||
},
|
||||
order: [ this.sequelize.random() ]
|
||||
}
|
||||
|
||||
return VideoModel.findAll(query)
|
||||
return VideoModel.scope({ method: [ ScopeNames.AVAILABLE_FOR_LIST_IDS, scopeOptions ] })
|
||||
.findAll(query)
|
||||
.then(rows => rows.map(r => r[ field ]))
|
||||
}
|
||||
|
||||
@ -1210,7 +1211,7 @@ export class VideoModel extends Model<VideoModel> {
|
||||
return {}
|
||||
}
|
||||
|
||||
private static async getAvailableForApi (query: IFindOptions<VideoModel>, options: AvailableForListIDsOptions) {
|
||||
private static async getAvailableForApi (query: IFindOptions<VideoModel>, options: AvailableForListIDsOptions, countVideos = true) {
|
||||
const idsScope = {
|
||||
method: [
|
||||
ScopeNames.AVAILABLE_FOR_LIST_IDS, options
|
||||
@ -1227,7 +1228,7 @@ export class VideoModel extends Model<VideoModel> {
|
||||
}
|
||||
|
||||
const [ count, rowsId ] = await Promise.all([
|
||||
VideoModel.scope(countScope).count(countQuery),
|
||||
countVideos ? VideoModel.scope(countScope).count(countQuery) : Promise.resolve(undefined),
|
||||
VideoModel.scope(idsScope).findAll(query)
|
||||
])
|
||||
const ids = rowsId.map(r => r.id)
|
||||
|
38
yarn.lock
38
yarn.lock
@ -160,6 +160,10 @@
|
||||
dependencies:
|
||||
"@types/node" "*"
|
||||
|
||||
"@types/memoizee@^0.4.2":
|
||||
version "0.4.2"
|
||||
resolved "https://registry.yarnpkg.com/@types/memoizee/-/memoizee-0.4.2.tgz#a500158999a8144a9b46cf9a9fb49b15f1853573"
|
||||
|
||||
"@types/mime@*":
|
||||
version "2.0.0"
|
||||
resolved "https://registry.yarnpkg.com/@types/mime/-/mime-2.0.0.tgz#5a7306e367c539b9f6543499de8dd519fac37a8b"
|
||||
@ -2058,7 +2062,7 @@ error@^7.0.0:
|
||||
string-template "~0.2.1"
|
||||
xtend "~4.0.0"
|
||||
|
||||
es5-ext@^0.10.14, es5-ext@^0.10.35, es5-ext@^0.10.9, es5-ext@~0.10.14:
|
||||
es5-ext@^0.10.14, es5-ext@^0.10.35, es5-ext@^0.10.45, es5-ext@^0.10.9, es5-ext@~0.10.14, es5-ext@~0.10.2:
|
||||
version "0.10.46"
|
||||
resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.46.tgz#efd99f67c5a7ec789baa3daa7f79870388f7f572"
|
||||
dependencies:
|
||||
@ -2110,7 +2114,7 @@ es6-symbol@3.1.1, es6-symbol@^3.1.1, es6-symbol@~3.1.1:
|
||||
d "1"
|
||||
es5-ext "~0.10.14"
|
||||
|
||||
es6-weak-map@^2.0.1:
|
||||
es6-weak-map@^2.0.1, es6-weak-map@^2.0.2:
|
||||
version "2.0.2"
|
||||
resolved "https://registry.yarnpkg.com/es6-weak-map/-/es6-weak-map-2.0.2.tgz#5e3ab32251ffd1538a1f8e5ffa1357772f92d96f"
|
||||
dependencies:
|
||||
@ -2223,7 +2227,7 @@ etag@~1.8.1:
|
||||
version "1.8.1"
|
||||
resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887"
|
||||
|
||||
event-emitter@~0.3.5:
|
||||
event-emitter@^0.3.5, event-emitter@~0.3.5:
|
||||
version "0.3.5"
|
||||
resolved "https://registry.yarnpkg.com/event-emitter/-/event-emitter-0.3.5.tgz#df8c69eef1647923c7157b9ce83840610b02cc39"
|
||||
dependencies:
|
||||
@ -3757,7 +3761,7 @@ is-plain-object@^2.0.1, is-plain-object@^2.0.3, is-plain-object@^2.0.4:
|
||||
dependencies:
|
||||
isobject "^3.0.1"
|
||||
|
||||
is-promise@^2.1.0:
|
||||
is-promise@^2.1, is-promise@^2.1.0:
|
||||
version "2.1.0"
|
||||
resolved "https://registry.yarnpkg.com/is-promise/-/is-promise-2.1.0.tgz#79a2a9ece7f096e80f36d2b2f3bc16c1ff4bf3fa"
|
||||
|
||||
@ -4490,6 +4494,12 @@ lru-cache@4.1.x, lru-cache@^4.0.1:
|
||||
pseudomap "^1.0.2"
|
||||
yallist "^2.1.2"
|
||||
|
||||
lru-queue@0.1:
|
||||
version "0.1.0"
|
||||
resolved "https://registry.yarnpkg.com/lru-queue/-/lru-queue-0.1.0.tgz#2738bd9f0d3cf4f84490c5736c48699ac632cda3"
|
||||
dependencies:
|
||||
es5-ext "~0.10.2"
|
||||
|
||||
lru@^3.0.0, lru@^3.1.0:
|
||||
version "3.1.0"
|
||||
resolved "https://registry.yarnpkg.com/lru/-/lru-3.1.0.tgz#ea7fb8546d83733396a13091d76cfeb4c06837d5"
|
||||
@ -4594,6 +4604,19 @@ mem@^1.1.0:
|
||||
dependencies:
|
||||
mimic-fn "^1.0.0"
|
||||
|
||||
memoizee@^0.4.14:
|
||||
version "0.4.14"
|
||||
resolved "https://registry.yarnpkg.com/memoizee/-/memoizee-0.4.14.tgz#07a00f204699f9a95c2d9e77218271c7cd610d57"
|
||||
dependencies:
|
||||
d "1"
|
||||
es5-ext "^0.10.45"
|
||||
es6-weak-map "^2.0.2"
|
||||
event-emitter "^0.3.5"
|
||||
is-promise "^2.1"
|
||||
lru-queue "0.1"
|
||||
next-tick "1"
|
||||
timers-ext "^0.1.5"
|
||||
|
||||
memory-chunk-store@^1.2.0:
|
||||
version "1.3.0"
|
||||
resolved "https://registry.yarnpkg.com/memory-chunk-store/-/memory-chunk-store-1.3.0.tgz#ae99e7e3b58b52db43d49d94722930d39459d0c4"
|
||||
@ -7201,6 +7224,13 @@ timed-out@^4.0.0:
|
||||
version "4.0.1"
|
||||
resolved "https://registry.yarnpkg.com/timed-out/-/timed-out-4.0.1.tgz#f32eacac5a175bea25d7fab565ab3ed8741ef56f"
|
||||
|
||||
timers-ext@^0.1.5:
|
||||
version "0.1.5"
|
||||
resolved "https://registry.yarnpkg.com/timers-ext/-/timers-ext-0.1.5.tgz#77147dd4e76b660c2abb8785db96574cbbd12922"
|
||||
dependencies:
|
||||
es5-ext "~0.10.14"
|
||||
next-tick "1"
|
||||
|
||||
tiny-lr@^1.1.1:
|
||||
version "1.1.1"
|
||||
resolved "https://registry.yarnpkg.com/tiny-lr/-/tiny-lr-1.1.1.tgz#9fa547412f238fedb068ee295af8b682c98b2aab"
|
||||
|
Loading…
Reference in New Issue
Block a user