Refractor and optimize AP collections

Only display urls in general object, and paginate video comments, shares, likes and
dislikes
This commit is contained in:
Chocobozzz 2018-05-25 16:21:16 +02:00
parent e251f170b0
commit 8fffe21a7b
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
14 changed files with 215 additions and 260 deletions

View File

@ -1,9 +1,8 @@
// Intercept ActivityPub client requests
import * as express from 'express'
import { VideoPrivacy } from '../../../shared/models/videos'
import { VideoPrivacy, VideoRateType } from '../../../shared/models/videos'
import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub'
import { pageToStartAndCount } from '../../helpers/core-utils'
import { ACTIVITY_PUB, CONFIG, ROUTE_CACHE_LIFETIME } from '../../initializers'
import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../initializers'
import { buildVideoAnnounce } from '../../lib/activitypub/send'
import { audiencify, getAudience } from '../../lib/activitypub/audience'
import { createActivityData } from '../../lib/activitypub/send/send-create'
@ -18,6 +17,14 @@ import { VideoChannelModel } from '../../models/video/video-channel'
import { VideoCommentModel } from '../../models/video/video-comment'
import { VideoShareModel } from '../../models/video/video-share'
import { cacheRoute } from '../../middlewares/cache'
import { activityPubResponse } from './utils'
import { AccountVideoRateModel } from '../../models/account/account-video-rate'
import {
getVideoCommentsActivityPubUrl,
getVideoDislikesActivityPubUrl,
getVideoLikesActivityPubUrl,
getVideoSharesActivityPubUrl
} from '../../lib/activitypub'
const activityPubClientRouter = express.Router()
@ -116,10 +123,8 @@ async function accountFollowingController (req: express.Request, res: express.Re
async function videoController (req: express.Request, res: express.Response, next: express.NextFunction) {
const video: VideoModel = res.locals.video
// We need more attributes
const videoAll = await VideoModel.loadAndPopulateAll(video.id)
const audience = await getAudience(video.VideoChannel.Account.Actor, undefined, video.privacy === VideoPrivacy.PUBLIC)
const videoObject = audiencify(videoAll.toActivityPubObject(), audience)
const videoObject = audiencify(video.toActivityPubObject(), audience)
if (req.path.endsWith('/activity')) {
const data = await createActivityData(video.url, video.VideoChannel.Account.Actor, videoObject, undefined, audience)
@ -139,41 +144,45 @@ async function videoAnnounceController (req: express.Request, res: express.Respo
async function videoAnnouncesController (req: express.Request, res: express.Response, next: express.NextFunction) {
const video: VideoModel = res.locals.video
// We need more attributes
const videoAll = await VideoModel.loadAndPopulateAll(video.id)
const object = videoAll.toAnnouncesActivityPubObject()
const handler = async (start: number, count: number) => {
const result = await VideoShareModel.listAndCountByVideoId(video.id, start, count)
return {
total: result.count,
data: result.rows.map(r => r.url)
}
}
const json = await activityPubCollectionPagination(getVideoSharesActivityPubUrl(video), handler, req.query.page)
return activityPubResponse(activityPubContextify(object), res)
return activityPubResponse(activityPubContextify(json), res)
}
async function videoLikesController (req: express.Request, res: express.Response, next: express.NextFunction) {
const video: VideoModel = res.locals.video
const json = await videoRates(req, 'like', video, getVideoLikesActivityPubUrl(video))
// We need more attributes
const videoAll = await VideoModel.loadAndPopulateAll(video.id)
const { likesObject } = videoAll.toRatesActivityPubObjects()
return activityPubResponse(activityPubContextify(likesObject), res)
return activityPubResponse(activityPubContextify(json), res)
}
async function videoDislikesController (req: express.Request, res: express.Response, next: express.NextFunction) {
const video: VideoModel = res.locals.video
const json = await videoRates(req, 'dislike', video, getVideoDislikesActivityPubUrl(video))
// We need more attributes
const videoAll = await VideoModel.loadAndPopulateAll(video.id)
const { dislikesObject } = videoAll.toRatesActivityPubObjects()
return activityPubResponse(activityPubContextify(dislikesObject), res)
return activityPubResponse(activityPubContextify(json), res)
}
async function videoCommentsController (req: express.Request, res: express.Response, next: express.NextFunction) {
const video: VideoModel = res.locals.video
// We need more attributes
const videoAll = await VideoModel.loadAndPopulateAll(video.id)
const commentsObject = videoAll.toCommentsActivityPubObject()
const handler = async (start: number, count: number) => {
const result = await VideoCommentModel.listAndCountByVideoId(video.id, start, count)
return {
total: result.count,
data: result.rows.map(r => r.url)
}
}
const json = await activityPubCollectionPagination(getVideoCommentsActivityPubUrl(video), handler, req.query.page)
return activityPubResponse(activityPubContextify(commentsObject), res)
return activityPubResponse(activityPubContextify(json), res)
}
async function videoChannelController (req: express.Request, res: express.Response, next: express.NextFunction) {
@ -216,23 +225,28 @@ async function videoCommentController (req: express.Request, res: express.Respon
// ---------------------------------------------------------------------------
async function actorFollowing (req: express.Request, actor: ActorModel) {
const page = req.query.page || 1
const { start, count } = pageToStartAndCount(page, ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE)
const handler = (start: number, count: number) => {
return ActorFollowModel.listAcceptedFollowingUrlsForApi([ actor.id ], undefined, start, count)
}
const result = await ActorFollowModel.listAcceptedFollowingUrlsForApi([ actor.id ], undefined, start, count)
return activityPubCollectionPagination(CONFIG.WEBSERVER.URL + req.url, page, result)
return activityPubCollectionPagination(CONFIG.WEBSERVER.URL + req.url, handler, req.query.page)
}
async function actorFollowers (req: express.Request, actor: ActorModel) {
const page = req.query.page || 1
const { start, count } = pageToStartAndCount(page, ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE)
const handler = (start: number, count: number) => {
return ActorFollowModel.listAcceptedFollowerUrlsForApi([ actor.id ], undefined, start, count)
}
const result = await ActorFollowModel.listAcceptedFollowerUrlsForApi([ actor.id ], undefined, start, count)
return activityPubCollectionPagination(CONFIG.WEBSERVER.URL + req.url, page, result)
return activityPubCollectionPagination(CONFIG.WEBSERVER.URL + req.url, handler, req.query.page)
}
function activityPubResponse (data: any, res: express.Response) {
return res.type('application/activity+json; charset=utf-8')
.json(data)
.end()
function videoRates (req: express.Request, rateType: VideoRateType, video: VideoModel, url: string) {
const handler = async (start: number, count: number) => {
const result = await AccountVideoRateModel.listAndCountAccountUrlsByVideoId(rateType, video.id, start, count)
return {
total: result.count,
data: result.rows.map(r => r.Account.Actor.url)
}
}
return activityPubCollectionPagination(url, handler, req.query.page)
}

View File

@ -1,16 +1,15 @@
import * as express from 'express'
import { Activity } from '../../../shared/models/activitypub/activity'
import { VideoPrivacy } from '../../../shared/models/videos'
import { activityPubCollectionPagination } from '../../helpers/activitypub'
import { pageToStartAndCount } from '../../helpers/core-utils'
import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub'
import { logger } from '../../helpers/logger'
import { ACTIVITY_PUB } from '../../initializers/constants'
import { announceActivityData, createActivityData } from '../../lib/activitypub/send'
import { buildAudience } from '../../lib/activitypub/audience'
import { asyncMiddleware, localAccountValidator } from '../../middlewares'
import { AccountModel } from '../../models/account/account'
import { ActorModel } from '../../models/activitypub/actor'
import { VideoModel } from '../../models/video/video'
import { activityPubResponse } from './utils'
const outboxRouter = express.Router()
@ -30,10 +29,17 @@ export {
async function outboxController (req: express.Request, res: express.Response, next: express.NextFunction) {
const account: AccountModel = res.locals.account
const actor = account.Actor
const actorOutboxUrl = account.Actor.url + '/outbox'
const page = req.query.page || 1
const { start, count } = pageToStartAndCount(page, ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE)
logger.info('Receiving outbox request for %s.', actorOutboxUrl)
const handler = (start: number, count: number) => buildActivities(actor, start, count)
const json = await activityPubCollectionPagination(actorOutboxUrl, handler, req.query.page)
return activityPubResponse(activityPubContextify(json), res)
}
async function buildActivities (actor: ActorModel, start: number, count: number) {
const data = await VideoModel.listAllAndSharedByActorForOutbox(actor.id, start, count)
const activities: Activity[] = []
@ -62,14 +68,8 @@ async function outboxController (req: express.Request, res: express.Response, ne
}
}
const newResult = {
return {
data: activities,
total: data.total
}
const actorOutboxUrl = account.Actor.url + '/outbox'
const json = activityPubCollectionPagination(actorOutboxUrl, page, newResult)
logger.info('Receiving outbox request for %s.', actorOutboxUrl)
return res.json(json).end()
}

View File

@ -0,0 +1,11 @@
import * as express from 'express'
function activityPubResponse (data: any, res: express.Response) {
return res.type('application/activity+json; charset=utf-8')
.json(data)
.end()
}
export {
activityPubResponse
}

View File

@ -1,8 +1,11 @@
import * as Bluebird from 'bluebird'
import * as validator from 'validator'
import { ResultList } from '../../shared/models'
import { Activity, ActivityPubActor } from '../../shared/models/activitypub'
import { ACTIVITY_PUB } from '../initializers'
import { ActorModel } from '../models/activitypub/actor'
import { signObject } from './peertube-crypto'
import { pageToStartAndCount } from './core-utils'
function activityPubContextify <T> (data: T) {
return Object.assign(data,{
@ -44,16 +47,23 @@ function activityPubContextify <T> (data: T) {
})
}
function activityPubCollection (url: string, results: any[]) {
return {
id: url,
type: 'OrderedCollection',
totalItems: results.length,
orderedItems: results
}
}
type ActivityPubCollectionPaginationHandler = (start: number, count: number) => Bluebird<ResultList<any>> | Promise<ResultList<any>>
async function activityPubCollectionPagination (url: string, handler: ActivityPubCollectionPaginationHandler, page?: any) {
if (!page || !validator.isInt(page)) {
// We just display the first page URL, we only need the total items
const result = await handler(0, 1)
return {
id: url,
type: 'OrderedCollection',
totalItems: result.total,
first: url + '?page=1'
}
}
const { start, count } = pageToStartAndCount(page, ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE)
const result = await handler(start, count)
function activityPubCollectionPagination (url: string, page: any, result: ResultList<any>) {
let next: string
let prev: string
@ -69,27 +79,16 @@ function activityPubCollectionPagination (url: string, page: any, result: Result
prev = url + '?page=' + (page - 1)
}
const orderedCollectionPagination = {
return {
id: url + '?page=' + page,
type: 'OrderedCollectionPage',
prev,
next,
partOf: url,
orderedItems: result.data
orderedItems: result.data,
totalItems: result.total
}
if (page === 1) {
return activityPubContextify({
id: url,
type: 'OrderedCollection',
totalItems: result.total,
first: orderedCollectionPagination
})
} else {
orderedCollectionPagination['totalItems'] = result.total
}
return orderedCollectionPagination
}
function buildSignedActivity (byActor: ActorModel, data: Object) {
@ -110,6 +109,5 @@ export {
getActorUrl,
activityPubContextify,
activityPubCollectionPagination,
activityPubCollection,
buildSignedActivity
}

View File

@ -0,0 +1,40 @@
import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../initializers'
import { doRequest } from '../../helpers/requests'
import { logger } from '../../helpers/logger'
async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Promise<any>) {
logger.info('Crawling ActivityPub data on %s.', uri)
const options = {
method: 'GET',
uri,
json: true,
activityPub: true,
timeout: JOB_REQUEST_TIMEOUT
}
const response = await doRequest(options)
const firstBody = response.body
let limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT
let i = 0
let nextLink = firstBody.first
while (nextLink && i < limit) {
options.uri = nextLink
const { body } = await doRequest(options)
nextLink = body.next
i++
if (Array.isArray(body.orderedItems)) {
const items = body.orderedItems
logger.info('Processing %i ActivityPub items for %s.', items.length, nextLink)
await handler(items)
}
}
}
export {
crawlCollectionPage
}

View File

@ -37,7 +37,7 @@ async function videoCommentActivityObjectToDBAttributes (video: VideoModel, acto
}
}
async function addVideoComments (instance: VideoModel, commentUrls: string[]) {
async function addVideoComments (commentUrls: string[], instance: VideoModel) {
for (const commentUrl of commentUrls) {
await addVideoComment(instance, commentUrl)
}

View File

@ -20,6 +20,7 @@ import { VideoFileModel } from '../../models/video/video-file'
import { VideoShareModel } from '../../models/video/video-share'
import { getOrCreateActorAndServerAndModel } from './actor'
import { addVideoComments } from './video-comments'
import { crawlCollectionPage } from './crawl'
function fetchRemoteVideoPreview (video: VideoModel, reject: Function) {
const host = video.VideoChannel.Account.Actor.Server.host
@ -216,25 +217,17 @@ async function getOrCreateAccountAndVideoAndChannel (videoObject: VideoTorrentOb
const video = await retryTransactionWrapper(getOrCreateVideo, options)
// Process outside the transaction because we could fetch remote data
if (videoObject.likes && Array.isArray(videoObject.likes.orderedItems)) {
logger.info('Adding likes of video %s.', video.uuid)
await createRates(videoObject.likes.orderedItems, video, 'like')
}
logger.info('Adding likes of video %s.', video.uuid)
await crawlCollectionPage<string>(videoObject.likes, (items) => createRates(items, video, 'like'))
if (videoObject.dislikes && Array.isArray(videoObject.dislikes.orderedItems)) {
logger.info('Adding dislikes of video %s.', video.uuid)
await createRates(videoObject.dislikes.orderedItems, video, 'dislike')
}
logger.info('Adding dislikes of video %s.', video.uuid)
await crawlCollectionPage<string>(videoObject.dislikes, (items) => createRates(items, video, 'dislike'))
if (videoObject.shares && Array.isArray(videoObject.shares.orderedItems)) {
logger.info('Adding shares of video %s.', video.uuid)
await addVideoShares(video, videoObject.shares.orderedItems)
}
logger.info('Adding shares of video %s.', video.uuid)
await crawlCollectionPage<string>(videoObject.shares, (items) => addVideoShares(items, video))
if (videoObject.comments && Array.isArray(videoObject.comments.orderedItems)) {
logger.info('Adding comments of video %s.', video.uuid)
await addVideoComments(video, videoObject.comments.orderedItems)
}
logger.info('Adding comments of video %s.', video.uuid)
await crawlCollectionPage<string>(videoObject.comments, (items) => addVideoComments(items, video))
return { actor, channelActor, video }
}
@ -266,7 +259,7 @@ async function createRates (actorUrls: string[], video: VideoModel, rate: VideoR
return
}
async function addVideoShares (instance: VideoModel, shareUrls: string[]) {
async function addVideoShares (shareUrls: string[], instance: VideoModel) {
for (const shareUrl of shareUrls) {
// Fetch url
const { body } = await doRequest({

View File

@ -1,9 +1,9 @@
import * as kue from 'kue'
import { logger } from '../../../helpers/logger'
import { doRequest } from '../../../helpers/requests'
import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../../initializers'
import { processActivities } from '../../activitypub/process'
import { ActivitypubHttpBroadcastPayload } from './activitypub-http-broadcast'
import { crawlCollectionPage } from '../../activitypub/crawl'
import { Activity } from '../../../../shared/models/activitypub'
export type ActivitypubHttpFetcherPayload = {
uris: string[]
@ -14,46 +14,8 @@ async function processActivityPubHttpFetcher (job: kue.Job) {
const payload = job.data as ActivitypubHttpBroadcastPayload
const options = {
method: 'GET',
uri: '',
json: true,
activityPub: true,
timeout: JOB_REQUEST_TIMEOUT
}
for (const uri of payload.uris) {
options.uri = uri
logger.info('Fetching ActivityPub data on %s.', uri)
const response = await doRequest(options)
const firstBody = response.body
if (firstBody.first && Array.isArray(firstBody.first.orderedItems)) {
const activities = firstBody.first.orderedItems
logger.info('Processing %i items ActivityPub fetcher for %s.', activities.length, options.uri)
await processActivities(activities)
}
let limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT
let i = 0
let nextLink = firstBody.first.next
while (nextLink && i < limit) {
options.uri = nextLink
const { body } = await doRequest(options)
nextLink = body.next
i++
if (Array.isArray(body.orderedItems)) {
const activities = body.orderedItems
logger.info('Processing %i items ActivityPub fetcher for %s.', activities.length, options.uri)
await processActivities(activities)
}
}
await crawlCollectionPage<Activity>(uri, (items) => processActivities(items))
}
}

View File

@ -6,6 +6,7 @@ import { VideoRateType } from '../../../shared/models/videos'
import { VIDEO_RATE_TYPES } from '../../initializers'
import { VideoModel } from '../video/video'
import { AccountModel } from './account'
import { ActorModel } from '../activitypub/actor'
/*
Account rates per video.
@ -66,4 +67,32 @@ export class AccountVideoRateModel extends Model<AccountVideoRateModel> {
return AccountVideoRateModel.findOne(options)
}
static listAndCountAccountUrlsByVideoId (rateType: VideoRateType, videoId: number, start: number, count: number, t?: Transaction) {
const query = {
start,
count,
where: {
videoId,
type: rateType
},
transaction: t,
include: [
{
attributes: [ 'actorId' ],
model: AccountModel.unscoped(),
required: true,
include: [
{
attributes: [ 'url' ],
model: ActorModel.unscoped(),
required: true
}
]
}
]
}
return AccountVideoRateModel.findAndCountAll(query)
}
}

View File

@ -335,8 +335,7 @@ export class ActorFollowModel extends Model<ActorFollowModel> {
tasks.push(ActorFollowModel.sequelize.query(query, options))
}
const [ followers, [ { total } ] ] = await
Promise.all(tasks)
const [ followers, [ { total } ] ] = await Promise.all(tasks)
const urls: string[] = followers.map(f => f.url)
return {

View File

@ -326,6 +326,20 @@ export class VideoCommentModel extends Model<VideoCommentModel> {
.findAll(query)
}
static listAndCountByVideoId (videoId: number, start: number, count: number, t?: Sequelize.Transaction, order: 'ASC' | 'DESC' = 'ASC') {
const query = {
order: [ [ 'createdAt', order ] ],
start,
count,
where: {
videoId
},
transaction: t
}
return VideoCommentModel.findAndCountAll(query)
}
static async getStats () {
const totalLocalVideoComments = await VideoCommentModel.count({
include: [

View File

@ -187,4 +187,17 @@ export class VideoShareModel extends Model<VideoShareModel> {
.findAll(query)
.then(res => res.map(r => r.Actor))
}
static listAndCountByVideoId (videoId: number, start: number, count: number, t?: Sequelize.Transaction) {
const query = {
start,
count,
where: {
videoId
},
transaction: t
}
return VideoShareModel.findAndCountAll(query)
}
}

View File

@ -29,7 +29,7 @@ import { VideoPrivacy, VideoResolution } from '../../../shared'
import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos'
import { VideoFilter } from '../../../shared/models/videos/video-query.type'
import { activityPubCollection } from '../../helpers/activitypub'
import { activityPubCollectionPagination } from '../../helpers/activitypub'
import {
createTorrentPromise,
peertubeTruncate,
@ -602,18 +602,6 @@ export class VideoModel extends Model<VideoModel> {
attributes: [ 'id', 'url' ],
model: VideoShareModel.unscoped(),
required: false,
where: {
[Sequelize.Op.and]: [
{
id: {
[Sequelize.Op.not]: null
}
},
{
actorId
}
]
},
include: [
{
attributes: [ 'id', 'url' ],
@ -644,35 +632,6 @@ export class VideoModel extends Model<VideoModel> {
}
]
},
{
attributes: [ 'type' ],
model: AccountVideoRateModel,
required: false,
include: [
{
attributes: [ 'id' ],
model: AccountModel.unscoped(),
include: [
{
attributes: [ 'url' ],
model: ActorModel.unscoped(),
include: [
{
attributes: [ 'host' ],
model: ServerModel,
required: false
}
]
}
]
}
]
},
{
attributes: [ 'url' ],
model: VideoCommentModel,
required: false
},
VideoFileModel,
TagModel
]
@ -897,26 +856,6 @@ export class VideoModel extends Model<VideoModel> {
.findOne(options)
}
static loadAndPopulateAll (id: number) {
const options = {
order: [ [ 'Tags', 'name', 'ASC' ] ],
where: {
id
}
}
return VideoModel
.scope([
ScopeNames.WITH_RATES,
ScopeNames.WITH_SHARES,
ScopeNames.WITH_TAGS,
ScopeNames.WITH_FILES,
ScopeNames.WITH_ACCOUNT_DETAILS,
ScopeNames.WITH_COMMENTS
])
.findOne(options)
}
static async getStats () {
const totalLocalVideos = await VideoModel.count({
where: {
@ -1203,25 +1142,6 @@ export class VideoModel extends Model<VideoModel> {
}
}
let likesObject
let dislikesObject
if (Array.isArray(this.AccountVideoRates)) {
const res = this.toRatesActivityPubObjects()
likesObject = res.likesObject
dislikesObject = res.dislikesObject
}
let sharesObject
if (Array.isArray(this.VideoShares)) {
sharesObject = this.toAnnouncesActivityPubObject()
}
let commentsObject
if (Array.isArray(this.VideoComments)) {
commentsObject = this.toCommentsActivityPubObject()
}
const url = []
for (const file of this.VideoFiles) {
url.push({
@ -1280,10 +1200,10 @@ export class VideoModel extends Model<VideoModel> {
height: THUMBNAILS_SIZE.height
},
url,
likes: likesObject,
dislikes: dislikesObject,
shares: sharesObject,
comments: commentsObject,
likes: getVideoLikesActivityPubUrl(this),
dislikes: getVideoDislikesActivityPubUrl(this),
shares: getVideoSharesActivityPubUrl(this),
comments: getVideoCommentsActivityPubUrl(this),
attributedTo: [
{
type: 'Person',
@ -1297,44 +1217,6 @@ export class VideoModel extends Model<VideoModel> {
}
}
toAnnouncesActivityPubObject () {
const shares: string[] = []
for (const videoShare of this.VideoShares) {
shares.push(videoShare.url)
}
return activityPubCollection(getVideoSharesActivityPubUrl(this), shares)
}
toCommentsActivityPubObject () {
const comments: string[] = []
for (const videoComment of this.VideoComments) {
comments.push(videoComment.url)
}
return activityPubCollection(getVideoCommentsActivityPubUrl(this), comments)
}
toRatesActivityPubObjects () {
const likes: string[] = []
const dislikes: string[] = []
for (const rate of this.AccountVideoRates) {
if (rate.type === 'like') {
likes.push(rate.Account.Actor.url)
} else if (rate.type === 'dislike') {
dislikes.push(rate.Account.Actor.url)
}
}
const likesObject = activityPubCollection(getVideoLikesActivityPubUrl(this), likes)
const dislikesObject = activityPubCollection(getVideoDislikesActivityPubUrl(this), dislikes)
return { likesObject, dislikesObject }
}
getTruncatedDescription () {
if (!this.description) return null

View File

@ -26,10 +26,10 @@ export interface VideoTorrentObject {
support: string
icon: ActivityIconObject
url: ActivityUrlObject[]
likes?: ActivityPubOrderedCollection<string>
dislikes?: ActivityPubOrderedCollection<string>
shares?: ActivityPubOrderedCollection<string>
comments?: ActivityPubOrderedCollection<string>
likes: string
dislikes: string
shares: string
comments: string
attributedTo: ActivityPubAttributedTo[]
to?: string[]
cc?: string[]