mirror of
https://github.com/Chocobozzz/PeerTube.git
synced 2024-08-17 17:30:32 +03:00
Basic video redundancy implementation
This commit is contained in:
parent
a651038487
commit
c48e82b5e0
@ -14,6 +14,8 @@ import { UserCreateComponent, UserListComponent, UsersComponent, UserService, Us
|
||||
import { ModerationCommentModalComponent, VideoAbuseListComponent, VideoBlacklistListComponent } from './moderation'
|
||||
import { UserBanModalComponent } from '@app/+admin/users/user-list/user-ban-modal.component'
|
||||
import { ModerationComponent } from '@app/+admin/moderation/moderation.component'
|
||||
import { RedundancyCheckboxComponent } from '@app/+admin/follows/shared/redundancy-checkbox.component'
|
||||
import { RedundancyService } from '@app/+admin/follows/shared/redundancy.service'
|
||||
|
||||
@NgModule({
|
||||
imports: [
|
||||
@ -29,6 +31,7 @@ import { ModerationComponent } from '@app/+admin/moderation/moderation.component
|
||||
FollowingAddComponent,
|
||||
FollowersListComponent,
|
||||
FollowingListComponent,
|
||||
RedundancyCheckboxComponent,
|
||||
|
||||
UsersComponent,
|
||||
UserCreateComponent,
|
||||
@ -54,6 +57,7 @@ import { ModerationComponent } from '@app/+admin/moderation/moderation.component
|
||||
|
||||
providers: [
|
||||
FollowService,
|
||||
RedundancyService,
|
||||
UserService,
|
||||
JobService,
|
||||
ConfigService
|
||||
|
@ -2,7 +2,7 @@ import { Component, OnInit } from '@angular/core'
|
||||
|
||||
import { NotificationsService } from 'angular2-notifications'
|
||||
import { SortMeta } from 'primeng/primeng'
|
||||
import { AccountFollow } from '../../../../../../shared/models/actors/follow.model'
|
||||
import { ActorFollow } from '../../../../../../shared/models/actors/follow.model'
|
||||
import { RestPagination, RestTable } from '../../../shared'
|
||||
import { FollowService } from '../shared'
|
||||
import { I18n } from '@ngx-translate/i18n-polyfill'
|
||||
@ -13,7 +13,7 @@ import { I18n } from '@ngx-translate/i18n-polyfill'
|
||||
styleUrls: [ './followers-list.component.scss' ]
|
||||
})
|
||||
export class FollowersListComponent extends RestTable implements OnInit {
|
||||
followers: AccountFollow[] = []
|
||||
followers: ActorFollow[] = []
|
||||
totalRecords = 0
|
||||
rowsPerPage = 10
|
||||
sort: SortMeta = { field: 'createdAt', order: 1 }
|
||||
|
@ -8,6 +8,7 @@
|
||||
<th i18n>Host</th>
|
||||
<th i18n>State</th>
|
||||
<th i18n pSortableColumn="createdAt">Created <p-sortIcon field="createdAt"></p-sortIcon></th>
|
||||
<th i18n>Redundancy allowed</th>
|
||||
<th></th>
|
||||
</tr>
|
||||
</ng-template>
|
||||
@ -18,6 +19,11 @@
|
||||
<td>{{ follow.following.host }}</td>
|
||||
<td>{{ follow.state }}</td>
|
||||
<td>{{ follow.createdAt }}</td>
|
||||
<td>
|
||||
<my-redundancy-checkbox
|
||||
[host]="follow.following.host" [redundancyAllowed]="follow.following.hostRedundancyAllowed"
|
||||
></my-redundancy-checkbox>
|
||||
</td>
|
||||
<td class="action-cell">
|
||||
<my-delete-button (click)="removeFollowing(follow)"></my-delete-button>
|
||||
</td>
|
||||
|
@ -0,0 +1,13 @@
|
||||
@import '_variables';
|
||||
@import '_mixins';
|
||||
|
||||
my-redundancy-checkbox /deep/ my-peertube-checkbox {
|
||||
.form-group {
|
||||
margin-bottom: 0;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
label {
|
||||
margin: 0;
|
||||
}
|
||||
}
|
@ -1,7 +1,7 @@
|
||||
import { Component, OnInit } from '@angular/core'
|
||||
import { NotificationsService } from 'angular2-notifications'
|
||||
import { SortMeta } from 'primeng/primeng'
|
||||
import { AccountFollow } from '../../../../../../shared/models/actors/follow.model'
|
||||
import { ActorFollow } from '../../../../../../shared/models/actors/follow.model'
|
||||
import { ConfirmService } from '../../../core/confirm/confirm.service'
|
||||
import { RestPagination, RestTable } from '../../../shared'
|
||||
import { FollowService } from '../shared'
|
||||
@ -9,10 +9,11 @@ import { I18n } from '@ngx-translate/i18n-polyfill'
|
||||
|
||||
@Component({
|
||||
selector: 'my-followers-list',
|
||||
templateUrl: './following-list.component.html'
|
||||
templateUrl: './following-list.component.html',
|
||||
styleUrls: [ './following-list.component.scss' ]
|
||||
})
|
||||
export class FollowingListComponent extends RestTable implements OnInit {
|
||||
following: AccountFollow[] = []
|
||||
following: ActorFollow[] = []
|
||||
totalRecords = 0
|
||||
rowsPerPage = 10
|
||||
sort: SortMeta = { field: 'createdAt', order: 1 }
|
||||
@ -31,7 +32,7 @@ export class FollowingListComponent extends RestTable implements OnInit {
|
||||
this.loadSort()
|
||||
}
|
||||
|
||||
async removeFollowing (follow: AccountFollow) {
|
||||
async removeFollowing (follow: ActorFollow) {
|
||||
const res = await this.confirmService.confirm(
|
||||
this.i18n('Do you really want to unfollow {{host}}?', { host: follow.following.host }),
|
||||
this.i18n('Unfollow')
|
||||
|
@ -3,7 +3,7 @@ import { HttpClient, HttpParams } from '@angular/common/http'
|
||||
import { Injectable } from '@angular/core'
|
||||
import { SortMeta } from 'primeng/primeng'
|
||||
import { Observable } from 'rxjs'
|
||||
import { AccountFollow, ResultList } from '../../../../../../shared'
|
||||
import { ActorFollow, ResultList } from '../../../../../../shared'
|
||||
import { environment } from '../../../../environments/environment'
|
||||
import { RestExtractor, RestPagination, RestService } from '../../../shared'
|
||||
|
||||
@ -18,22 +18,22 @@ export class FollowService {
|
||||
) {
|
||||
}
|
||||
|
||||
getFollowing (pagination: RestPagination, sort: SortMeta): Observable<ResultList<AccountFollow>> {
|
||||
getFollowing (pagination: RestPagination, sort: SortMeta): Observable<ResultList<ActorFollow>> {
|
||||
let params = new HttpParams()
|
||||
params = this.restService.addRestGetParams(params, pagination, sort)
|
||||
|
||||
return this.authHttp.get<ResultList<AccountFollow>>(FollowService.BASE_APPLICATION_URL + '/following', { params })
|
||||
return this.authHttp.get<ResultList<ActorFollow>>(FollowService.BASE_APPLICATION_URL + '/following', { params })
|
||||
.pipe(
|
||||
map(res => this.restExtractor.convertResultListDateToHuman(res)),
|
||||
catchError(res => this.restExtractor.handleError(res))
|
||||
)
|
||||
}
|
||||
|
||||
getFollowers (pagination: RestPagination, sort: SortMeta): Observable<ResultList<AccountFollow>> {
|
||||
getFollowers (pagination: RestPagination, sort: SortMeta): Observable<ResultList<ActorFollow>> {
|
||||
let params = new HttpParams()
|
||||
params = this.restService.addRestGetParams(params, pagination, sort)
|
||||
|
||||
return this.authHttp.get<ResultList<AccountFollow>>(FollowService.BASE_APPLICATION_URL + '/followers', { params })
|
||||
return this.authHttp.get<ResultList<ActorFollow>>(FollowService.BASE_APPLICATION_URL + '/followers', { params })
|
||||
.pipe(
|
||||
map(res => this.restExtractor.convertResultListDateToHuman(res)),
|
||||
catchError(res => this.restExtractor.handleError(res))
|
||||
@ -52,7 +52,7 @@ export class FollowService {
|
||||
)
|
||||
}
|
||||
|
||||
unfollow (follow: AccountFollow) {
|
||||
unfollow (follow: ActorFollow) {
|
||||
return this.authHttp.delete(FollowService.BASE_APPLICATION_URL + '/following/' + follow.following.host)
|
||||
.pipe(
|
||||
map(this.restExtractor.extractDataBool),
|
||||
|
@ -0,0 +1,3 @@
|
||||
<my-peertube-checkbox
|
||||
[inputName]="host + '-redundancy-allowed'" [(ngModel)]="redundancyAllowed" (ngModelChange)="updateRedundancyState()"
|
||||
></my-peertube-checkbox>
|
@ -0,0 +1,2 @@
|
||||
@import '_variables';
|
||||
@import '_mixins';
|
@ -0,0 +1,42 @@
|
||||
import { Component, Input } from '@angular/core'
|
||||
import { AuthService } from '@app/core'
|
||||
import { RestExtractor } from '@app/shared/rest'
|
||||
import { RedirectService } from '@app/core/routing/redirect.service'
|
||||
import { NotificationsService } from 'angular2-notifications'
|
||||
import { I18n } from '@ngx-translate/i18n-polyfill'
|
||||
import { RedundancyService } from '@app/+admin/follows/shared/redundancy.service'
|
||||
|
||||
@Component({
|
||||
selector: 'my-redundancy-checkbox',
|
||||
templateUrl: './redundancy-checkbox.component.html',
|
||||
styleUrls: [ './redundancy-checkbox.component.scss' ]
|
||||
})
|
||||
export class RedundancyCheckboxComponent {
|
||||
@Input() redundancyAllowed: boolean
|
||||
@Input() host: string
|
||||
|
||||
constructor (
|
||||
private authService: AuthService,
|
||||
private restExtractor: RestExtractor,
|
||||
private redirectService: RedirectService,
|
||||
private notificationsService: NotificationsService,
|
||||
private redundancyService: RedundancyService,
|
||||
private i18n: I18n
|
||||
) { }
|
||||
|
||||
updateRedundancyState () {
|
||||
this.redundancyService.updateRedundancy(this.host, this.redundancyAllowed)
|
||||
.subscribe(
|
||||
() => {
|
||||
const stateLabel = this.redundancyAllowed ? this.i18n('enabled') : this.i18n('disabled')
|
||||
|
||||
this.notificationsService.success(
|
||||
this.i18n('Success'),
|
||||
this.i18n('Redundancy for {{host}} is {{stateLabel}}', { host: this.host, stateLabel })
|
||||
)
|
||||
},
|
||||
|
||||
err => this.notificationsService.error(this.i18n('Error'), err.message)
|
||||
)
|
||||
}
|
||||
}
|
29
client/src/app/+admin/follows/shared/redundancy.service.ts
Normal file
29
client/src/app/+admin/follows/shared/redundancy.service.ts
Normal file
@ -0,0 +1,29 @@
|
||||
import { catchError, map } from 'rxjs/operators'
|
||||
import { HttpClient } from '@angular/common/http'
|
||||
import { Injectable } from '@angular/core'
|
||||
import { RestExtractor, RestService } from '@app/shared'
|
||||
import { environment } from '../../../../environments/environment'
|
||||
|
||||
@Injectable()
|
||||
export class RedundancyService {
|
||||
static BASE_USER_SUBSCRIPTIONS_URL = environment.apiUrl + '/api/v1/server/redundancy'
|
||||
|
||||
constructor (
|
||||
private authHttp: HttpClient,
|
||||
private restExtractor: RestExtractor,
|
||||
private restService: RestService
|
||||
) { }
|
||||
|
||||
updateRedundancy (host: string, redundancyAllowed: boolean) {
|
||||
const url = RedundancyService.BASE_USER_SUBSCRIPTIONS_URL + '/' + host
|
||||
|
||||
const body = { redundancyAllowed }
|
||||
|
||||
return this.authHttp.put(url, body)
|
||||
.pipe(
|
||||
map(this.restExtractor.extractDataBool),
|
||||
catchError(err => this.restExtractor.handleError(err))
|
||||
)
|
||||
}
|
||||
|
||||
}
|
@ -66,6 +66,15 @@ trending:
|
||||
videos:
|
||||
interval_days: 7 # Compute trending videos for the last x days
|
||||
|
||||
# Cache remote videos on your server, to help other instances to broadcast the video
|
||||
# You can define multiple caches using different sizes/strategies
|
||||
# Once you have defined your strategies, choose which instances you want to cache in admin -> manage follows -> following
|
||||
redundancy:
|
||||
videos:
|
||||
# -
|
||||
# size: '10GB'
|
||||
# strategy: 'most-views' # Cache videos that have the most views
|
||||
|
||||
cache:
|
||||
previews:
|
||||
size: 500 # Max number of previews you want to cache
|
||||
|
@ -67,6 +67,15 @@ trending:
|
||||
videos:
|
||||
interval_days: 7 # Compute trending videos for the last x days
|
||||
|
||||
# Cache remote videos on your server, to help other instances to broadcast the video
|
||||
# You can define multiple caches using different sizes/strategies
|
||||
# Once you have defined your strategies, choose which instances you want to cache in admin -> manage follows -> following
|
||||
redundancy:
|
||||
videos:
|
||||
# -
|
||||
# size: '10GB'
|
||||
# strategy: 'most-views' # Cache videos that have the most views
|
||||
|
||||
###############################################################################
|
||||
#
|
||||
# From this point, all the following keys can be overridden by the web interface
|
||||
|
@ -21,6 +21,12 @@ smtp:
|
||||
log:
|
||||
level: 'debug'
|
||||
|
||||
redundancy:
|
||||
videos:
|
||||
-
|
||||
size: '100KB'
|
||||
strategy: 'most-views'
|
||||
|
||||
cache:
|
||||
previews:
|
||||
size: 1
|
||||
|
@ -86,6 +86,7 @@
|
||||
"bluebird": "^3.5.0",
|
||||
"body-parser": "^1.12.4",
|
||||
"bull": "^3.4.2",
|
||||
"bytes": "^3.0.0",
|
||||
"commander": "^2.13.0",
|
||||
"concurrently": "^4.0.1",
|
||||
"config": "^2.0.1",
|
||||
@ -145,6 +146,7 @@
|
||||
"@types/bluebird": "3.5.21",
|
||||
"@types/body-parser": "^1.16.3",
|
||||
"@types/bull": "^3.3.12",
|
||||
"@types/bytes": "^3.0.0",
|
||||
"@types/chai": "^4.0.4",
|
||||
"@types/chai-json-schema": "^1.4.3",
|
||||
"@types/chai-xml": "^0.3.1",
|
||||
|
@ -94,6 +94,7 @@ import { BadActorFollowScheduler } from './server/lib/schedulers/bad-actor-follo
|
||||
import { RemoveOldJobsScheduler } from './server/lib/schedulers/remove-old-jobs-scheduler'
|
||||
import { UpdateVideosScheduler } from './server/lib/schedulers/update-videos-scheduler'
|
||||
import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-update-scheduler'
|
||||
import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler'
|
||||
|
||||
// ----------- Command line -----------
|
||||
|
||||
@ -206,6 +207,7 @@ async function startApplication () {
|
||||
RemoveOldJobsScheduler.Instance.enable()
|
||||
UpdateVideosScheduler.Instance.enable()
|
||||
YoutubeDlUpdateScheduler.Instance.enable()
|
||||
VideosRedundancyScheduler.Instance.enable()
|
||||
|
||||
// Redis initialization
|
||||
Redis.Instance.init()
|
||||
|
@ -3,9 +3,9 @@ import * as express from 'express'
|
||||
import { VideoPrivacy, VideoRateType } from '../../../shared/models/videos'
|
||||
import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub'
|
||||
import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../initializers'
|
||||
import { buildVideoAnnounce } from '../../lib/activitypub/send'
|
||||
import { buildAnnounceWithVideoAudience } from '../../lib/activitypub/send'
|
||||
import { audiencify, getAudience } from '../../lib/activitypub/audience'
|
||||
import { createActivityData } from '../../lib/activitypub/send/send-create'
|
||||
import { buildCreateActivity } from '../../lib/activitypub/send/send-create'
|
||||
import { asyncMiddleware, executeIfActivityPub, localAccountValidator, localVideoChannelValidator } from '../../middlewares'
|
||||
import { videosGetValidator, videosShareValidator } from '../../middlewares/validators'
|
||||
import { videoCommentGetValidator } from '../../middlewares/validators/video-comments'
|
||||
@ -26,6 +26,8 @@ import {
|
||||
getVideoSharesActivityPubUrl
|
||||
} from '../../lib/activitypub'
|
||||
import { VideoCaptionModel } from '../../models/video/video-caption'
|
||||
import { videoRedundancyGetValidator } from '../../middlewares/validators/redundancy'
|
||||
import { getServerActor } from '../../helpers/utils'
|
||||
|
||||
const activityPubClientRouter = express.Router()
|
||||
|
||||
@ -93,6 +95,11 @@ activityPubClientRouter.get('/video-channels/:name/following',
|
||||
executeIfActivityPub(asyncMiddleware(videoChannelFollowingController))
|
||||
)
|
||||
|
||||
activityPubClientRouter.get('/redundancy/videos/:videoId/:resolution([0-9]+)(-:fps([0-9]+))?',
|
||||
executeIfActivityPub(asyncMiddleware(videoRedundancyGetValidator)),
|
||||
executeIfActivityPub(asyncMiddleware(videoRedundancyController))
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
@ -131,7 +138,7 @@ async function videoController (req: express.Request, res: express.Response, nex
|
||||
const videoObject = audiencify(video.toActivityPubObject(), audience)
|
||||
|
||||
if (req.path.endsWith('/activity')) {
|
||||
const data = createActivityData(video.url, video.VideoChannel.Account.Actor, videoObject, audience)
|
||||
const data = buildCreateActivity(video.url, video.VideoChannel.Account.Actor, videoObject, audience)
|
||||
return activityPubResponse(activityPubContextify(data), res)
|
||||
}
|
||||
|
||||
@ -140,9 +147,9 @@ async function videoController (req: express.Request, res: express.Response, nex
|
||||
|
||||
async function videoAnnounceController (req: express.Request, res: express.Response, next: express.NextFunction) {
|
||||
const share = res.locals.videoShare as VideoShareModel
|
||||
const object = await buildVideoAnnounce(share.Actor, share, res.locals.video, undefined)
|
||||
const { activity } = await buildAnnounceWithVideoAudience(share.Actor, share, res.locals.video, undefined)
|
||||
|
||||
return activityPubResponse(activityPubContextify(object), res)
|
||||
return activityPubResponse(activityPubContextify(activity), res)
|
||||
}
|
||||
|
||||
async function videoAnnouncesController (req: express.Request, res: express.Response, next: express.NextFunction) {
|
||||
@ -219,13 +226,28 @@ async function videoCommentController (req: express.Request, res: express.Respon
|
||||
const videoCommentObject = audiencify(videoComment.toActivityPubObject(threadParentComments), audience)
|
||||
|
||||
if (req.path.endsWith('/activity')) {
|
||||
const data = createActivityData(videoComment.url, videoComment.Account.Actor, videoCommentObject, audience)
|
||||
const data = buildCreateActivity(videoComment.url, videoComment.Account.Actor, videoCommentObject, audience)
|
||||
return activityPubResponse(activityPubContextify(data), res)
|
||||
}
|
||||
|
||||
return activityPubResponse(activityPubContextify(videoCommentObject), res)
|
||||
}
|
||||
|
||||
async function videoRedundancyController (req: express.Request, res: express.Response) {
|
||||
const videoRedundancy = res.locals.videoRedundancy
|
||||
const serverActor = await getServerActor()
|
||||
|
||||
const audience = getAudience(serverActor)
|
||||
const object = audiencify(videoRedundancy.toActivityPubObject(), audience)
|
||||
|
||||
if (req.path.endsWith('/activity')) {
|
||||
const data = buildCreateActivity(videoRedundancy.url, serverActor, object, audience)
|
||||
return activityPubResponse(activityPubContextify(data), res)
|
||||
}
|
||||
|
||||
return activityPubResponse(activityPubContextify(object), res)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function actorFollowing (req: express.Request, actor: ActorModel) {
|
||||
|
@ -3,7 +3,7 @@ import { Activity } from '../../../shared/models/activitypub/activity'
|
||||
import { VideoPrivacy } from '../../../shared/models/videos'
|
||||
import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub'
|
||||
import { logger } from '../../helpers/logger'
|
||||
import { announceActivityData, createActivityData } from '../../lib/activitypub/send'
|
||||
import { buildAnnounceActivity, buildCreateActivity } from '../../lib/activitypub/send'
|
||||
import { buildAudience } from '../../lib/activitypub/audience'
|
||||
import { asyncMiddleware, localAccountValidator, localVideoChannelValidator } from '../../middlewares'
|
||||
import { AccountModel } from '../../models/account/account'
|
||||
@ -60,12 +60,12 @@ async function buildActivities (actor: ActorModel, start: number, count: number)
|
||||
// This is a shared video
|
||||
if (video.VideoShares !== undefined && video.VideoShares.length !== 0) {
|
||||
const videoShare = video.VideoShares[0]
|
||||
const announceActivity = announceActivityData(videoShare.url, actor, video.url, createActivityAudience)
|
||||
const announceActivity = buildAnnounceActivity(videoShare.url, actor, video.url, createActivityAudience)
|
||||
|
||||
activities.push(announceActivity)
|
||||
} else {
|
||||
const videoObject = video.toActivityPubObject()
|
||||
const createActivity = createActivityData(video.url, byActor, videoObject, createActivityAudience)
|
||||
const createActivity = buildCreateActivity(video.url, byActor, videoObject, createActivityAudience)
|
||||
|
||||
activities.push(createActivity)
|
||||
}
|
||||
|
@ -17,8 +17,6 @@ import {
|
||||
import { VideoChannelsSearchQuery, VideosSearchQuery } from '../../../shared/models/search'
|
||||
import { getOrCreateActorAndServerAndModel, getOrCreateVideoAndAccountAndChannel } from '../../lib/activitypub'
|
||||
import { logger } from '../../helpers/logger'
|
||||
import { User } from '../../../shared/models/users'
|
||||
import { CONFIG } from '../../initializers/constants'
|
||||
import { VideoChannelModel } from '../../models/video/video-channel'
|
||||
import { loadActorUrlOrGetFromWebfinger } from '../../helpers/webfinger'
|
||||
|
||||
|
@ -96,6 +96,11 @@ async function removeFollow (req: express.Request, res: express.Response, next:
|
||||
await sequelizeTypescript.transaction(async t => {
|
||||
if (follow.state === 'accepted') await sendUndoFollow(follow, t)
|
||||
|
||||
// Disable redundancy on unfollowed instances
|
||||
const server = follow.ActorFollowing.Server
|
||||
server.redundancyAllowed = false
|
||||
await server.save({ transaction: t })
|
||||
|
||||
await follow.destroy({ transaction: t })
|
||||
})
|
||||
|
||||
|
@ -1,10 +1,12 @@
|
||||
import * as express from 'express'
|
||||
import { serverFollowsRouter } from './follows'
|
||||
import { statsRouter } from './stats'
|
||||
import { serverRedundancyRouter } from './redundancy'
|
||||
|
||||
const serverRouter = express.Router()
|
||||
|
||||
serverRouter.use('/', serverFollowsRouter)
|
||||
serverRouter.use('/', serverRedundancyRouter)
|
||||
serverRouter.use('/', statsRouter)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
32
server/controllers/api/server/redundancy.ts
Normal file
32
server/controllers/api/server/redundancy.ts
Normal file
@ -0,0 +1,32 @@
|
||||
import * as express from 'express'
|
||||
import { UserRight } from '../../../../shared/models/users'
|
||||
import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares'
|
||||
import { updateServerRedundancyValidator } from '../../../middlewares/validators/redundancy'
|
||||
import { ServerModel } from '../../../models/server/server'
|
||||
|
||||
const serverRedundancyRouter = express.Router()
|
||||
|
||||
serverRedundancyRouter.put('/redundancy/:host',
|
||||
authenticate,
|
||||
ensureUserHasRight(UserRight.MANAGE_SERVER_FOLLOW),
|
||||
asyncMiddleware(updateServerRedundancyValidator),
|
||||
asyncMiddleware(updateRedundancy)
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
serverRedundancyRouter
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function updateRedundancy (req: express.Request, res: express.Response, next: express.NextFunction) {
|
||||
const server = res.locals.server as ServerModel
|
||||
|
||||
server.redundancyAllowed = req.body.redundancyAllowed
|
||||
|
||||
await server.save()
|
||||
|
||||
return res.sendStatus(204)
|
||||
}
|
@ -112,7 +112,7 @@ async function reportVideoAbuse (req: express.Request, res: express.Response) {
|
||||
|
||||
// We send the video abuse to the origin server
|
||||
if (videoInstance.isOwned() === false) {
|
||||
await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance, t)
|
||||
await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance)
|
||||
}
|
||||
|
||||
auditLogger.create(reporterAccount.Actor.getIdentifier(), new VideoAbuseAuditView(videoAbuseInstance.toFormattedJSON()))
|
||||
|
@ -14,20 +14,24 @@ function activityPubContextify <T> (data: T) {
|
||||
'https://w3id.org/security/v1',
|
||||
{
|
||||
RsaSignature2017: 'https://w3id.org/security#RsaSignature2017',
|
||||
pt: 'https://joinpeertube.org/ns',
|
||||
schema: 'http://schema.org#',
|
||||
Hashtag: 'as:Hashtag',
|
||||
uuid: 'http://schema.org/identifier',
|
||||
category: 'http://schema.org/category',
|
||||
licence: 'http://schema.org/license',
|
||||
subtitleLanguage: 'http://schema.org/subtitleLanguage',
|
||||
uuid: 'schema:identifier',
|
||||
category: 'schema:category',
|
||||
licence: 'schema:license',
|
||||
subtitleLanguage: 'schema:subtitleLanguage',
|
||||
sensitive: 'as:sensitive',
|
||||
language: 'http://schema.org/inLanguage',
|
||||
views: 'http://schema.org/Number',
|
||||
stats: 'http://schema.org/Number',
|
||||
size: 'http://schema.org/Number',
|
||||
fps: 'http://schema.org/Number',
|
||||
commentsEnabled: 'http://schema.org/Boolean',
|
||||
waitTranscoding: 'http://schema.org/Boolean',
|
||||
support: 'http://schema.org/Text'
|
||||
language: 'schema:inLanguage',
|
||||
views: 'schema:Number',
|
||||
stats: 'schema:Number',
|
||||
size: 'schema:Number',
|
||||
fps: 'schema:Number',
|
||||
commentsEnabled: 'schema:Boolean',
|
||||
waitTranscoding: 'schema:Boolean',
|
||||
expires: 'schema:expires',
|
||||
support: 'schema:Text',
|
||||
CacheFile: 'pt:CacheFile'
|
||||
},
|
||||
{
|
||||
likes: {
|
||||
|
@ -1,7 +1,10 @@
|
||||
import * as validator from 'validator'
|
||||
import { Activity, ActivityType } from '../../../../shared/models/activitypub'
|
||||
import {
|
||||
isActorAcceptActivityValid, isActorDeleteActivityValid, isActorFollowActivityValid, isActorRejectActivityValid,
|
||||
isActorAcceptActivityValid,
|
||||
isActorDeleteActivityValid,
|
||||
isActorFollowActivityValid,
|
||||
isActorRejectActivityValid,
|
||||
isActorUpdateActivityValid
|
||||
} from './actor'
|
||||
import { isAnnounceActivityValid } from './announce'
|
||||
@ -11,12 +14,13 @@ import { isUndoActivityValid } from './undo'
|
||||
import { isVideoCommentCreateActivityValid, isVideoCommentDeleteActivityValid } from './video-comments'
|
||||
import {
|
||||
isVideoFlagValid,
|
||||
sanitizeAndCheckVideoTorrentCreateActivity,
|
||||
isVideoTorrentDeleteActivityValid,
|
||||
sanitizeAndCheckVideoTorrentCreateActivity,
|
||||
sanitizeAndCheckVideoTorrentUpdateActivity
|
||||
} from './videos'
|
||||
import { isViewActivityValid } from './view'
|
||||
import { exists } from '../misc'
|
||||
import { isCacheFileCreateActivityValid, isCacheFileUpdateActivityValid } from './cache-file'
|
||||
|
||||
function isRootActivityValid (activity: any) {
|
||||
return Array.isArray(activity['@context']) && (
|
||||
@ -67,11 +71,13 @@ function checkCreateActivity (activity: any) {
|
||||
isDislikeActivityValid(activity) ||
|
||||
sanitizeAndCheckVideoTorrentCreateActivity(activity) ||
|
||||
isVideoFlagValid(activity) ||
|
||||
isVideoCommentCreateActivityValid(activity)
|
||||
isVideoCommentCreateActivityValid(activity) ||
|
||||
isCacheFileCreateActivityValid(activity)
|
||||
}
|
||||
|
||||
function checkUpdateActivity (activity: any) {
|
||||
return sanitizeAndCheckVideoTorrentUpdateActivity(activity) ||
|
||||
return isCacheFileUpdateActivityValid(activity) ||
|
||||
sanitizeAndCheckVideoTorrentUpdateActivity(activity) ||
|
||||
isActorUpdateActivityValid(activity)
|
||||
}
|
||||
|
||||
|
28
server/helpers/custom-validators/activitypub/cache-file.ts
Normal file
28
server/helpers/custom-validators/activitypub/cache-file.ts
Normal file
@ -0,0 +1,28 @@
|
||||
import { isActivityPubUrlValid, isBaseActivityValid } from './misc'
|
||||
import { isRemoteVideoUrlValid } from './videos'
|
||||
import { isDateValid, exists } from '../misc'
|
||||
import { CacheFileObject } from '../../../../shared/models/activitypub/objects'
|
||||
|
||||
function isCacheFileCreateActivityValid (activity: any) {
|
||||
return isBaseActivityValid(activity, 'Create') &&
|
||||
isCacheFileObjectValid(activity.object)
|
||||
}
|
||||
|
||||
function isCacheFileUpdateActivityValid (activity: any) {
|
||||
return isBaseActivityValid(activity, 'Update') &&
|
||||
isCacheFileObjectValid(activity.object)
|
||||
}
|
||||
|
||||
function isCacheFileObjectValid (object: CacheFileObject) {
|
||||
return exists(object) &&
|
||||
object.type === 'CacheFile' &&
|
||||
isDateValid(object.expires) &&
|
||||
isActivityPubUrlValid(object.object) &&
|
||||
isRemoteVideoUrlValid(object.url)
|
||||
}
|
||||
|
||||
export {
|
||||
isCacheFileUpdateActivityValid,
|
||||
isCacheFileCreateActivityValid,
|
||||
isCacheFileObjectValid
|
||||
}
|
@ -3,7 +3,7 @@ import { CONSTRAINTS_FIELDS } from '../../../initializers'
|
||||
import { isTestInstance } from '../../core-utils'
|
||||
import { exists } from '../misc'
|
||||
|
||||
function isActivityPubUrlValid (url: string) {
|
||||
function isUrlValid (url: string) {
|
||||
const isURLOptions = {
|
||||
require_host: true,
|
||||
require_tld: true,
|
||||
@ -17,13 +17,18 @@ function isActivityPubUrlValid (url: string) {
|
||||
isURLOptions.require_tld = false
|
||||
}
|
||||
|
||||
return exists(url) && validator.isURL('' + url, isURLOptions) && validator.isLength('' + url, CONSTRAINTS_FIELDS.ACTORS.URL)
|
||||
return exists(url) && validator.isURL('' + url, isURLOptions)
|
||||
}
|
||||
|
||||
function isActivityPubUrlValid (url: string) {
|
||||
return isUrlValid(url) && validator.isLength('' + url, CONSTRAINTS_FIELDS.ACTORS.URL)
|
||||
}
|
||||
|
||||
function isBaseActivityValid (activity: any, type: string) {
|
||||
return (activity['@context'] === undefined || Array.isArray(activity['@context'])) &&
|
||||
activity.type === type &&
|
||||
isActivityPubUrlValid(activity.id) &&
|
||||
exists(activity.actor) &&
|
||||
(isActivityPubUrlValid(activity.actor) || isActivityPubUrlValid(activity.actor.id)) &&
|
||||
(
|
||||
activity.to === undefined ||
|
||||
@ -49,6 +54,7 @@ function setValidAttributedTo (obj: any) {
|
||||
}
|
||||
|
||||
export {
|
||||
isUrlValid,
|
||||
isActivityPubUrlValid,
|
||||
isBaseActivityValid,
|
||||
setValidAttributedTo
|
||||
|
@ -2,6 +2,7 @@ import { isActorFollowActivityValid } from './actor'
|
||||
import { isBaseActivityValid } from './misc'
|
||||
import { isDislikeActivityValid, isLikeActivityValid } from './rate'
|
||||
import { isAnnounceActivityValid } from './announce'
|
||||
import { isCacheFileCreateActivityValid } from './cache-file'
|
||||
|
||||
function isUndoActivityValid (activity: any) {
|
||||
return isBaseActivityValid(activity, 'Undo') &&
|
||||
@ -9,7 +10,8 @@ function isUndoActivityValid (activity: any) {
|
||||
isActorFollowActivityValid(activity.object) ||
|
||||
isLikeActivityValid(activity.object) ||
|
||||
isDislikeActivityValid(activity.object) ||
|
||||
isAnnounceActivityValid(activity.object)
|
||||
isAnnounceActivityValid(activity.object) ||
|
||||
isCacheFileCreateActivityValid(activity.object)
|
||||
)
|
||||
}
|
||||
|
||||
|
@ -75,6 +75,30 @@ function sanitizeAndCheckVideoTorrentObject (video: any) {
|
||||
video.attributedTo.length !== 0
|
||||
}
|
||||
|
||||
function isRemoteVideoUrlValid (url: any) {
|
||||
// FIXME: Old bug, we used the width to represent the resolution. Remove it in a few release (currently beta.11)
|
||||
if (url.width && !url.height) url.height = url.width
|
||||
|
||||
return url.type === 'Link' &&
|
||||
(
|
||||
ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.indexOf(url.mimeType) !== -1 &&
|
||||
isActivityPubUrlValid(url.href) &&
|
||||
validator.isInt(url.height + '', { min: 0 }) &&
|
||||
validator.isInt(url.size + '', { min: 0 }) &&
|
||||
(!url.fps || validator.isInt(url.fps + '', { min: 0 }))
|
||||
) ||
|
||||
(
|
||||
ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.indexOf(url.mimeType) !== -1 &&
|
||||
isActivityPubUrlValid(url.href) &&
|
||||
validator.isInt(url.height + '', { min: 0 })
|
||||
) ||
|
||||
(
|
||||
ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mimeType) !== -1 &&
|
||||
validator.isLength(url.href, { min: 5 }) &&
|
||||
validator.isInt(url.height + '', { min: 0 })
|
||||
)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
@ -83,7 +107,8 @@ export {
|
||||
isVideoTorrentDeleteActivityValid,
|
||||
isRemoteStringIdentifierValid,
|
||||
isVideoFlagValid,
|
||||
sanitizeAndCheckVideoTorrentObject
|
||||
sanitizeAndCheckVideoTorrentObject,
|
||||
isRemoteVideoUrlValid
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@ -147,26 +172,4 @@ function setRemoteVideoTruncatedContent (video: any) {
|
||||
return true
|
||||
}
|
||||
|
||||
function isRemoteVideoUrlValid (url: any) {
|
||||
// FIXME: Old bug, we used the width to represent the resolution. Remove it in a few realease (currently beta.11)
|
||||
if (url.width && !url.height) url.height = url.width
|
||||
|
||||
return url.type === 'Link' &&
|
||||
(
|
||||
ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.indexOf(url.mimeType) !== -1 &&
|
||||
isActivityPubUrlValid(url.href) &&
|
||||
validator.isInt(url.height + '', { min: 0 }) &&
|
||||
validator.isInt(url.size + '', { min: 0 }) &&
|
||||
(!url.fps || validator.isInt(url.fps + '', { min: 0 }))
|
||||
) ||
|
||||
(
|
||||
ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.indexOf(url.mimeType) !== -1 &&
|
||||
isActivityPubUrlValid(url.href) &&
|
||||
validator.isInt(url.height + '', { min: 0 })
|
||||
) ||
|
||||
(
|
||||
ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mimeType) !== -1 &&
|
||||
validator.isLength(url.href, { min: 5 }) &&
|
||||
validator.isInt(url.height + '', { min: 0 })
|
||||
)
|
||||
}
|
||||
|
@ -5,44 +5,49 @@ import { createWriteStream, remove } from 'fs-extra'
|
||||
import { CONFIG } from '../initializers'
|
||||
import { join } from 'path'
|
||||
|
||||
function downloadWebTorrentVideo (target: { magnetUri: string, torrentName: string }) {
|
||||
function downloadWebTorrentVideo (target: { magnetUri: string, torrentName?: string }, timeout?: number) {
|
||||
const id = target.magnetUri || target.torrentName
|
||||
let timer
|
||||
|
||||
const path = generateVideoTmpPath(id)
|
||||
logger.info('Importing torrent video %s', id)
|
||||
|
||||
return new Promise<string>((res, rej) => {
|
||||
const webtorrent = new WebTorrent()
|
||||
let file: WebTorrent.TorrentFile
|
||||
|
||||
const torrentId = target.magnetUri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName)
|
||||
|
||||
const options = { path: CONFIG.STORAGE.VIDEOS_DIR }
|
||||
const torrent = webtorrent.add(torrentId, options, torrent => {
|
||||
if (torrent.files.length !== 1) return rej(new Error('The number of files is not equal to 1 for ' + torrentId))
|
||||
if (torrent.files.length !== 1) {
|
||||
if (timer) clearTimeout(timer)
|
||||
|
||||
const file = torrent.files[ 0 ]
|
||||
return safeWebtorrentDestroy(webtorrent, torrentId, file.name, target.torrentName)
|
||||
.then(() => rej(new Error('The number of files is not equal to 1 for ' + torrentId)))
|
||||
}
|
||||
|
||||
file = torrent.files[ 0 ]
|
||||
|
||||
const writeStream = createWriteStream(path)
|
||||
writeStream.on('finish', () => {
|
||||
webtorrent.destroy(async err => {
|
||||
if (err) return rej(err)
|
||||
if (timer) clearTimeout(timer)
|
||||
|
||||
if (target.torrentName) {
|
||||
remove(torrentId)
|
||||
.catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentId, { err }))
|
||||
}
|
||||
|
||||
remove(join(CONFIG.STORAGE.VIDEOS_DIR, file.name))
|
||||
.catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', file.name, { err }))
|
||||
|
||||
res(path)
|
||||
})
|
||||
return safeWebtorrentDestroy(webtorrent, torrentId, file.name, target.torrentName)
|
||||
.then(() => res(path))
|
||||
})
|
||||
|
||||
file.createReadStream().pipe(writeStream)
|
||||
})
|
||||
|
||||
torrent.on('error', err => rej(err))
|
||||
|
||||
if (timeout) {
|
||||
timer = setTimeout(async () => {
|
||||
return safeWebtorrentDestroy(webtorrent, torrentId, file ? file.name : undefined, target.torrentName)
|
||||
.then(() => rej(new Error('Webtorrent download timeout.')))
|
||||
}, timeout)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@ -51,3 +56,29 @@ function downloadWebTorrentVideo (target: { magnetUri: string, torrentName: stri
|
||||
export {
|
||||
downloadWebTorrentVideo
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function safeWebtorrentDestroy (webtorrent: WebTorrent.Instance, torrentId: string, filename?: string, torrentName?: string) {
|
||||
return new Promise(res => {
|
||||
webtorrent.destroy(err => {
|
||||
// Delete torrent file
|
||||
if (torrentName) {
|
||||
remove(torrentId)
|
||||
.catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentId, { err }))
|
||||
}
|
||||
|
||||
// Delete downloaded file
|
||||
if (filename) {
|
||||
remove(join(CONFIG.STORAGE.VIDEOS_DIR, filename))
|
||||
.catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', filename, { err }))
|
||||
}
|
||||
|
||||
if (err) {
|
||||
logger.warn('Cannot destroy webtorrent in timeout.', { err })
|
||||
}
|
||||
|
||||
return res()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
@ -7,6 +7,9 @@ import { parse } from 'url'
|
||||
import { CONFIG } from './constants'
|
||||
import { logger } from '../helpers/logger'
|
||||
import { getServerActor } from '../helpers/utils'
|
||||
import { VideosRedundancy } from '../../shared/models/redundancy'
|
||||
import { isArray } from '../helpers/custom-validators/misc'
|
||||
import { uniq } from 'lodash'
|
||||
|
||||
async function checkActivityPubUrls () {
|
||||
const actor = await getServerActor()
|
||||
@ -35,6 +38,20 @@ function checkConfig () {
|
||||
return 'NSFW policy setting should be "do_not_list" or "blur" or "display" instead of ' + defaultNSFWPolicy
|
||||
}
|
||||
|
||||
const redundancyVideos = config.get<VideosRedundancy[]>('redundancy.videos')
|
||||
if (isArray(redundancyVideos)) {
|
||||
for (const r of redundancyVideos) {
|
||||
if ([ 'most-views' ].indexOf(r.strategy) === -1) {
|
||||
return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy
|
||||
}
|
||||
}
|
||||
|
||||
const filtered = uniq(redundancyVideos.map(r => r.strategy))
|
||||
if (filtered.length !== redundancyVideos.length) {
|
||||
return 'Redundancy video entries should have uniq strategies'
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
|
@ -1,6 +1,6 @@
|
||||
import { IConfig } from 'config'
|
||||
import { dirname, join } from 'path'
|
||||
import { JobType, VideoRateType, VideoState } from '../../shared/models'
|
||||
import { JobType, VideoRateType, VideoRedundancyStrategy, VideoState, VideosRedundancy } from '../../shared/models'
|
||||
import { ActivityPubActorType } from '../../shared/models/activitypub'
|
||||
import { FollowState } from '../../shared/models/actors'
|
||||
import { VideoAbuseState, VideoImportState, VideoPrivacy } from '../../shared/models/videos'
|
||||
@ -9,13 +9,14 @@ import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../h
|
||||
import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
|
||||
import { invert } from 'lodash'
|
||||
import { CronRepeatOptions, EveryRepeatOptions } from 'bull'
|
||||
import * as bytes from 'bytes'
|
||||
|
||||
// Use a variable to reload the configuration if we need
|
||||
let config: IConfig = require('config')
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const LAST_MIGRATION_VERSION = 265
|
||||
const LAST_MIGRATION_VERSION = 270
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@ -137,7 +138,8 @@ let SCHEDULER_INTERVALS_MS = {
|
||||
badActorFollow: 60000 * 60, // 1 hour
|
||||
removeOldJobs: 60000 * 60, // 1 hour
|
||||
updateVideos: 60000, // 1 minute
|
||||
youtubeDLUpdate: 60000 * 60 * 24 // 1 day
|
||||
youtubeDLUpdate: 60000 * 60 * 24, // 1 day
|
||||
videosRedundancy: 60000 * 2 // 2 hours
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@ -208,6 +210,9 @@ const CONFIG = {
|
||||
INTERVAL_DAYS: config.get<number>('trending.videos.interval_days')
|
||||
}
|
||||
},
|
||||
REDUNDANCY: {
|
||||
VIDEOS: buildVideosRedundancy(config.get<any[]>('redundancy.videos'))
|
||||
},
|
||||
ADMIN: {
|
||||
get EMAIL () { return config.get<string>('admin.email') }
|
||||
},
|
||||
@ -321,6 +326,9 @@ const CONSTRAINTS_FIELDS = {
|
||||
}
|
||||
}
|
||||
},
|
||||
VIDEOS_REDUNDANCY: {
|
||||
URL: { min: 3, max: 2000 } // Length
|
||||
},
|
||||
VIDEOS: {
|
||||
NAME: { min: 3, max: 120 }, // Length
|
||||
LANGUAGE: { min: 1, max: 10 }, // Length
|
||||
@ -584,6 +592,13 @@ const CACHE = {
|
||||
}
|
||||
}
|
||||
|
||||
const REDUNDANCY = {
|
||||
VIDEOS: {
|
||||
EXPIRES_AFTER_MS: 48 * 3600 * 1000, // 2 days
|
||||
RANDOMIZED_FACTOR: 5
|
||||
}
|
||||
}
|
||||
|
||||
const ACCEPT_HEADERS = [ 'html', 'application/json' ].concat(ACTIVITY_PUB.POTENTIAL_ACCEPT_HEADERS)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@ -629,8 +644,11 @@ if (isTestInstance() === true) {
|
||||
SCHEDULER_INTERVALS_MS.badActorFollow = 10000
|
||||
SCHEDULER_INTERVALS_MS.removeOldJobs = 10000
|
||||
SCHEDULER_INTERVALS_MS.updateVideos = 5000
|
||||
SCHEDULER_INTERVALS_MS.videosRedundancy = 5000
|
||||
REPEAT_JOBS['videos-views'] = { every: 5000 }
|
||||
|
||||
REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1
|
||||
|
||||
VIDEO_VIEW_LIFETIME = 1000 // 1 second
|
||||
|
||||
JOB_ATTEMPTS['email'] = 1
|
||||
@ -653,6 +671,7 @@ export {
|
||||
CONFIG,
|
||||
CONSTRAINTS_FIELDS,
|
||||
EMBED_SIZE,
|
||||
REDUNDANCY,
|
||||
JOB_CONCURRENCY,
|
||||
JOB_ATTEMPTS,
|
||||
LAST_MIGRATION_VERSION,
|
||||
@ -722,6 +741,17 @@ function updateWebserverConfig () {
|
||||
CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP)
|
||||
}
|
||||
|
||||
function buildVideosRedundancy (objs: { strategy: VideoRedundancyStrategy, size: string }[]): VideosRedundancy[] {
|
||||
if (!objs) return []
|
||||
|
||||
return objs.map(obj => {
|
||||
return {
|
||||
strategy: obj.strategy,
|
||||
size: bytes.parse(obj.size)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function buildLanguages () {
|
||||
const iso639 = require('iso-639-3')
|
||||
|
||||
|
@ -27,6 +27,7 @@ import { VideoCaptionModel } from '../models/video/video-caption'
|
||||
import { VideoImportModel } from '../models/video/video-import'
|
||||
import { VideoViewModel } from '../models/video/video-views'
|
||||
import { VideoChangeOwnershipModel } from '../models/video/video-change-ownership'
|
||||
import { VideoRedundancyModel } from '../models/redundancy/video-redundancy'
|
||||
|
||||
require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
|
||||
|
||||
@ -87,7 +88,8 @@ async function initDatabaseModels (silent: boolean) {
|
||||
VideoCommentModel,
|
||||
ScheduleVideoUpdateModel,
|
||||
VideoImportModel,
|
||||
VideoViewModel
|
||||
VideoViewModel,
|
||||
VideoRedundancyModel
|
||||
])
|
||||
|
||||
// Check extensions exist in the database
|
||||
|
24
server/initializers/migrations/0270-server-redundancy.ts
Normal file
24
server/initializers/migrations/0270-server-redundancy.ts
Normal file
@ -0,0 +1,24 @@
|
||||
import * as Sequelize from 'sequelize'
|
||||
|
||||
async function up (utils: {
|
||||
transaction: Sequelize.Transaction
|
||||
queryInterface: Sequelize.QueryInterface
|
||||
sequelize: Sequelize.Sequelize
|
||||
}): Promise<any> {
|
||||
{
|
||||
const data = {
|
||||
type: Sequelize.BOOLEAN,
|
||||
allowNull: false,
|
||||
defaultValue: false
|
||||
}
|
||||
|
||||
await utils.queryInterface.addColumn('server', 'redundancyAllowed', data)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function down (options) {
|
||||
throw new Error('Not implemented.')
|
||||
}
|
||||
|
||||
export { up, down }
|
@ -400,17 +400,15 @@ async function refreshActorIfNeeded (actor: ActorModel): Promise<{ actor: ActorM
|
||||
await actor.save({ transaction: t })
|
||||
|
||||
if (actor.Account) {
|
||||
await actor.save({ transaction: t })
|
||||
|
||||
actor.Account.set('name', result.name)
|
||||
actor.Account.set('description', result.summary)
|
||||
|
||||
await actor.Account.save({ transaction: t })
|
||||
} else if (actor.VideoChannel) {
|
||||
await actor.save({ transaction: t })
|
||||
|
||||
actor.VideoChannel.set('name', result.name)
|
||||
actor.VideoChannel.set('description', result.summary)
|
||||
actor.VideoChannel.set('support', result.support)
|
||||
|
||||
await actor.VideoChannel.save({ transaction: t })
|
||||
}
|
||||
|
||||
|
47
server/lib/activitypub/cache-file.ts
Normal file
47
server/lib/activitypub/cache-file.ts
Normal file
@ -0,0 +1,47 @@
|
||||
import { CacheFileObject } from '../../../shared/index'
|
||||
import { VideoModel } from '../../models/video/video'
|
||||
import { ActorModel } from '../../models/activitypub/actor'
|
||||
import { sequelizeTypescript } from '../../initializers'
|
||||
import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
|
||||
|
||||
function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: ActorModel) {
|
||||
const url = cacheFileObject.url
|
||||
|
||||
const videoFile = video.VideoFiles.find(f => {
|
||||
return f.resolution === url.height && f.fps === url.fps
|
||||
})
|
||||
|
||||
if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`)
|
||||
|
||||
return {
|
||||
expiresOn: new Date(cacheFileObject.expires),
|
||||
url: cacheFileObject.id,
|
||||
fileUrl: cacheFileObject.url.href,
|
||||
strategy: null,
|
||||
videoFileId: videoFile.id,
|
||||
actorId: byActor.id
|
||||
}
|
||||
}
|
||||
|
||||
function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, byActor: ActorModel) {
|
||||
return sequelizeTypescript.transaction(async t => {
|
||||
const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
|
||||
|
||||
return VideoRedundancyModel.create(attributes, { transaction: t })
|
||||
})
|
||||
}
|
||||
|
||||
function updateCacheFile (cacheFileObject: CacheFileObject, redundancyModel: VideoRedundancyModel, byActor: ActorModel) {
|
||||
const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, redundancyModel.VideoFile.Video, byActor)
|
||||
|
||||
redundancyModel.set('expires', attributes.expiresOn)
|
||||
redundancyModel.set('fileUrl', attributes.fileUrl)
|
||||
|
||||
return redundancyModel.save()
|
||||
}
|
||||
|
||||
export {
|
||||
createCacheFile,
|
||||
updateCacheFile,
|
||||
cacheFileActivityObjectToDBAttributes
|
||||
}
|
@ -1,4 +1,4 @@
|
||||
import { ActivityCreate, VideoAbuseState, VideoTorrentObject } from '../../../../shared'
|
||||
import { ActivityCreate, CacheFileObject, VideoAbuseState, VideoTorrentObject } from '../../../../shared'
|
||||
import { DislikeObject, VideoAbuseObject, ViewObject } from '../../../../shared/models/activitypub/objects'
|
||||
import { VideoCommentObject } from '../../../../shared/models/activitypub/objects/video-comment-object'
|
||||
import { retryTransactionWrapper } from '../../../helpers/database-utils'
|
||||
@ -12,6 +12,7 @@ import { addVideoComment, resolveThread } from '../video-comments'
|
||||
import { getOrCreateVideoAndAccountAndChannel } from '../videos'
|
||||
import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils'
|
||||
import { Redis } from '../../redis'
|
||||
import { createCacheFile } from '../cache-file'
|
||||
|
||||
async function processCreateActivity (activity: ActivityCreate) {
|
||||
const activityObject = activity.object
|
||||
@ -28,6 +29,8 @@ async function processCreateActivity (activity: ActivityCreate) {
|
||||
return retryTransactionWrapper(processCreateVideoAbuse, actor, activityObject as VideoAbuseObject)
|
||||
} else if (activityType === 'Note') {
|
||||
return retryTransactionWrapper(processCreateVideoComment, actor, activity)
|
||||
} else if (activityType === 'CacheFile') {
|
||||
return retryTransactionWrapper(processCacheFile, actor, activity)
|
||||
}
|
||||
|
||||
logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id })
|
||||
@ -97,6 +100,20 @@ async function processCreateView (byActor: ActorModel, activity: ActivityCreate)
|
||||
}
|
||||
}
|
||||
|
||||
async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) {
|
||||
const cacheFile = activity.object as CacheFileObject
|
||||
|
||||
const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFile.object)
|
||||
|
||||
await createCacheFile(cacheFile, video, byActor)
|
||||
|
||||
if (video.isOwned()) {
|
||||
// Don't resend the activity to the sender
|
||||
const exceptions = [ byActor ]
|
||||
await forwardActivity(activity, undefined, exceptions)
|
||||
}
|
||||
}
|
||||
|
||||
async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) {
|
||||
logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object)
|
||||
|
||||
@ -113,7 +130,7 @@ async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateDat
|
||||
state: VideoAbuseState.PENDING
|
||||
}
|
||||
|
||||
await VideoAbuseModel.create(videoAbuseData)
|
||||
await VideoAbuseModel.create(videoAbuseData, { transaction: t })
|
||||
|
||||
logger.info('Remote abuse for video uuid %s created', videoAbuseToCreateData.object)
|
||||
})
|
||||
|
@ -1,4 +1,4 @@
|
||||
import { ActivityAnnounce, ActivityFollow, ActivityLike, ActivityUndo } from '../../../../shared/models/activitypub'
|
||||
import { ActivityAnnounce, ActivityFollow, ActivityLike, ActivityUndo, CacheFileObject } from '../../../../shared/models/activitypub'
|
||||
import { DislikeObject } from '../../../../shared/models/activitypub/objects'
|
||||
import { getActorUrl } from '../../../helpers/activitypub'
|
||||
import { retryTransactionWrapper } from '../../../helpers/database-utils'
|
||||
@ -11,6 +11,7 @@ import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
|
||||
import { forwardVideoRelatedActivity } from '../send/utils'
|
||||
import { getOrCreateVideoAndAccountAndChannel } from '../videos'
|
||||
import { VideoShareModel } from '../../../models/video/video-share'
|
||||
import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
|
||||
|
||||
async function processUndoActivity (activity: ActivityUndo) {
|
||||
const activityToUndo = activity.object
|
||||
@ -19,11 +20,21 @@ async function processUndoActivity (activity: ActivityUndo) {
|
||||
|
||||
if (activityToUndo.type === 'Like') {
|
||||
return retryTransactionWrapper(processUndoLike, actorUrl, activity)
|
||||
} else if (activityToUndo.type === 'Create' && activityToUndo.object.type === 'Dislike') {
|
||||
return retryTransactionWrapper(processUndoDislike, actorUrl, activity)
|
||||
} else if (activityToUndo.type === 'Follow') {
|
||||
}
|
||||
|
||||
if (activityToUndo.type === 'Create') {
|
||||
if (activityToUndo.object.type === 'Dislike') {
|
||||
return retryTransactionWrapper(processUndoDislike, actorUrl, activity)
|
||||
} else if (activityToUndo.object.type === 'CacheFile') {
|
||||
return retryTransactionWrapper(processUndoCacheFile, actorUrl, activity)
|
||||
}
|
||||
}
|
||||
|
||||
if (activityToUndo.type === 'Follow') {
|
||||
return retryTransactionWrapper(processUndoFollow, actorUrl, activityToUndo)
|
||||
} else if (activityToUndo.type === 'Announce') {
|
||||
}
|
||||
|
||||
if (activityToUndo.type === 'Announce') {
|
||||
return retryTransactionWrapper(processUndoAnnounce, actorUrl, activityToUndo)
|
||||
}
|
||||
|
||||
@ -88,6 +99,29 @@ async function processUndoDislike (actorUrl: string, activity: ActivityUndo) {
|
||||
})
|
||||
}
|
||||
|
||||
async function processUndoCacheFile (actorUrl: string, activity: ActivityUndo) {
|
||||
const cacheFileObject = activity.object.object as CacheFileObject
|
||||
|
||||
const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.object)
|
||||
|
||||
return sequelizeTypescript.transaction(async t => {
|
||||
const byActor = await ActorModel.loadByUrl(actorUrl)
|
||||
if (!byActor) throw new Error('Unknown actor ' + actorUrl)
|
||||
|
||||
const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id)
|
||||
if (!cacheFile) throw new Error('Unknown video cache ' + cacheFile.url)
|
||||
|
||||
await cacheFile.destroy()
|
||||
|
||||
if (video.isOwned()) {
|
||||
// Don't resend the activity to the sender
|
||||
const exceptions = [ byActor ]
|
||||
|
||||
await forwardVideoRelatedActivity(activity, t, exceptions, video)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function processUndoFollow (actorUrl: string, followActivity: ActivityFollow) {
|
||||
return sequelizeTypescript.transaction(async t => {
|
||||
const follower = await ActorModel.loadByUrl(actorUrl, t)
|
||||
|
@ -1,4 +1,4 @@
|
||||
import { ActivityUpdate, VideoTorrentObject } from '../../../../shared/models/activitypub'
|
||||
import { ActivityUpdate, CacheFileObject, VideoTorrentObject } from '../../../../shared/models/activitypub'
|
||||
import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor'
|
||||
import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils'
|
||||
import { logger } from '../../../helpers/logger'
|
||||
@ -7,8 +7,11 @@ import { AccountModel } from '../../../models/account/account'
|
||||
import { ActorModel } from '../../../models/activitypub/actor'
|
||||
import { VideoChannelModel } from '../../../models/video/video-channel'
|
||||
import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor'
|
||||
import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, updateVideoFromAP } from '../videos'
|
||||
import { getOrCreateVideoAndAccountAndChannel, updateVideoFromAP, getOrCreateVideoChannelFromVideoObject } from '../videos'
|
||||
import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos'
|
||||
import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file'
|
||||
import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
|
||||
import { createCacheFile, updateCacheFile } from '../cache-file'
|
||||
|
||||
async function processUpdateActivity (activity: ActivityUpdate) {
|
||||
const actor = await getOrCreateActorAndServerAndModel(activity.actor)
|
||||
@ -16,10 +19,16 @@ async function processUpdateActivity (activity: ActivityUpdate) {
|
||||
|
||||
if (objectType === 'Video') {
|
||||
return retryTransactionWrapper(processUpdateVideo, actor, activity)
|
||||
} else if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') {
|
||||
}
|
||||
|
||||
if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') {
|
||||
return retryTransactionWrapper(processUpdateActor, actor, activity)
|
||||
}
|
||||
|
||||
if (objectType === 'CacheFile') {
|
||||
return retryTransactionWrapper(processUpdateCacheFile, actor, activity)
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
@ -42,7 +51,24 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate)
|
||||
const { video } = await getOrCreateVideoAndAccountAndChannel(videoObject.id)
|
||||
const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
|
||||
|
||||
return updateVideoFromAP(video, videoObject, actor, channelActor, activity.to)
|
||||
return updateVideoFromAP(video, videoObject, actor.Account, channelActor.VideoChannel, activity.to)
|
||||
}
|
||||
|
||||
async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUpdate) {
|
||||
const cacheFileObject = activity.object as CacheFileObject
|
||||
|
||||
if (!isCacheFileObjectValid(cacheFileObject) === false) {
|
||||
logger.debug('Cahe file object sent by update is not valid.', { cacheFileObject })
|
||||
return undefined
|
||||
}
|
||||
|
||||
const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id)
|
||||
if (!redundancyModel) {
|
||||
const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.id)
|
||||
return createCacheFile(cacheFileObject, video, byActor)
|
||||
}
|
||||
|
||||
return updateCacheFile(cacheFileObject, redundancyModel, byActor)
|
||||
}
|
||||
|
||||
async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) {
|
||||
|
@ -3,7 +3,7 @@ import { ActorModel } from '../../../models/activitypub/actor'
|
||||
import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
|
||||
import { getActorFollowAcceptActivityPubUrl, getActorFollowActivityPubUrl } from '../url'
|
||||
import { unicastTo } from './utils'
|
||||
import { followActivityData } from './send-follow'
|
||||
import { buildFollowActivity } from './send-follow'
|
||||
import { logger } from '../../../helpers/logger'
|
||||
|
||||
async function sendAccept (actorFollow: ActorFollowModel) {
|
||||
@ -18,10 +18,10 @@ async function sendAccept (actorFollow: ActorFollowModel) {
|
||||
logger.info('Creating job to accept follower %s.', follower.url)
|
||||
|
||||
const followUrl = getActorFollowActivityPubUrl(actorFollow)
|
||||
const followData = followActivityData(followUrl, follower, me)
|
||||
const followData = buildFollowActivity(followUrl, follower, me)
|
||||
|
||||
const url = getActorFollowAcceptActivityPubUrl(actorFollow)
|
||||
const data = acceptActivityData(url, me, followData)
|
||||
const data = buildAcceptActivity(url, me, followData)
|
||||
|
||||
return unicastTo(data, me, follower.inboxUrl)
|
||||
}
|
||||
@ -34,7 +34,7 @@ export {
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function acceptActivityData (url: string, byActor: ActorModel, followActivityData: ActivityFollow): ActivityAccept {
|
||||
function buildAcceptActivity (url: string, byActor: ActorModel, followActivityData: ActivityFollow): ActivityAccept {
|
||||
return {
|
||||
type: 'Accept',
|
||||
id: url,
|
||||
|
@ -4,45 +4,44 @@ import { ActorModel } from '../../../models/activitypub/actor'
|
||||
import { VideoModel } from '../../../models/video/video'
|
||||
import { VideoShareModel } from '../../../models/video/video-share'
|
||||
import { broadcastToFollowers } from './utils'
|
||||
import { getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience'
|
||||
import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience'
|
||||
import { logger } from '../../../helpers/logger'
|
||||
|
||||
async function buildVideoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
|
||||
async function buildAnnounceWithVideoAudience (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
|
||||
const announcedObject = video.url
|
||||
|
||||
const accountsToForwardView = await getActorsInvolvedInVideo(video, t)
|
||||
const audience = getObjectFollowersAudience(accountsToForwardView)
|
||||
return announceActivityData(videoShare.url, byActor, announcedObject, audience)
|
||||
const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
|
||||
const audience = getObjectFollowersAudience(actorsInvolvedInVideo)
|
||||
|
||||
const activity = buildAnnounceActivity(videoShare.url, byActor, announcedObject, audience)
|
||||
|
||||
return { activity, actorsInvolvedInVideo }
|
||||
}
|
||||
|
||||
async function sendVideoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
|
||||
const data = await buildVideoAnnounce(byActor, videoShare, video, t)
|
||||
const { activity, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, t)
|
||||
|
||||
logger.info('Creating job to send announce %s.', videoShare.url)
|
||||
|
||||
const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
|
||||
const followersException = [ byActor ]
|
||||
|
||||
return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, followersException)
|
||||
return broadcastToFollowers(activity, byActor, actorsInvolvedInVideo, t, followersException)
|
||||
}
|
||||
|
||||
function announceActivityData (url: string, byActor: ActorModel, object: string, audience?: ActivityAudience): ActivityAnnounce {
|
||||
function buildAnnounceActivity (url: string, byActor: ActorModel, object: string, audience?: ActivityAudience): ActivityAnnounce {
|
||||
if (!audience) audience = getAudience(byActor)
|
||||
|
||||
return {
|
||||
type: 'Announce',
|
||||
to: audience.to,
|
||||
cc: audience.cc,
|
||||
return audiencify({
|
||||
type: 'Announce' as 'Announce',
|
||||
id: url,
|
||||
actor: byActor.url,
|
||||
object
|
||||
}
|
||||
}, audience)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
sendVideoAnnounce,
|
||||
announceActivityData,
|
||||
buildVideoAnnounce
|
||||
buildAnnounceActivity,
|
||||
buildAnnounceWithVideoAudience
|
||||
}
|
||||
|
@ -17,6 +17,7 @@ import {
|
||||
getVideoCommentAudience
|
||||
} from '../audience'
|
||||
import { logger } from '../../../helpers/logger'
|
||||
import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
|
||||
|
||||
async function sendCreateVideo (video: VideoModel, t: Transaction) {
|
||||
if (video.privacy === VideoPrivacy.PRIVATE) return undefined
|
||||
@ -27,12 +28,12 @@ async function sendCreateVideo (video: VideoModel, t: Transaction) {
|
||||
const videoObject = video.toActivityPubObject()
|
||||
|
||||
const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC)
|
||||
const data = createActivityData(video.url, byActor, videoObject, audience)
|
||||
const createActivity = buildCreateActivity(video.url, byActor, videoObject, audience)
|
||||
|
||||
return broadcastToFollowers(data, byActor, [ byActor ], t)
|
||||
return broadcastToFollowers(createActivity, byActor, [ byActor ], t)
|
||||
}
|
||||
|
||||
async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel, t: Transaction) {
|
||||
async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) {
|
||||
if (!video.VideoChannel.Account.Actor.serverId) return // Local
|
||||
|
||||
const url = getVideoAbuseActivityPubUrl(videoAbuse)
|
||||
@ -40,9 +41,23 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel,
|
||||
logger.info('Creating job to send video abuse %s.', url)
|
||||
|
||||
const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] }
|
||||
const data = createActivityData(url, byActor, videoAbuse.toActivityPubObject(), audience)
|
||||
const createActivity = buildCreateActivity(url, byActor, videoAbuse.toActivityPubObject(), audience)
|
||||
|
||||
return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
|
||||
return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
|
||||
}
|
||||
|
||||
async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) {
|
||||
logger.info('Creating job to send file cache of %s.', fileRedundancy.url)
|
||||
|
||||
const redundancyObject = fileRedundancy.toActivityPubObject()
|
||||
|
||||
const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id)
|
||||
const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, undefined)
|
||||
|
||||
const audience = getVideoAudience(video, actorsInvolvedInVideo)
|
||||
const createActivity = buildCreateActivity(fileRedundancy.url, byActor, redundancyObject, audience)
|
||||
|
||||
return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
|
||||
}
|
||||
|
||||
async function sendCreateVideoComment (comment: VideoCommentModel, t: Transaction) {
|
||||
@ -66,73 +81,73 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio
|
||||
audience = getObjectFollowersAudience(actorsInvolvedInComment.concat(parentsCommentActors))
|
||||
}
|
||||
|
||||
const data = createActivityData(comment.url, byActor, commentObject, audience)
|
||||
const createActivity = buildCreateActivity(comment.url, byActor, commentObject, audience)
|
||||
|
||||
// This was a reply, send it to the parent actors
|
||||
const actorsException = [ byActor ]
|
||||
await broadcastToActors(data, byActor, parentsCommentActors, actorsException)
|
||||
await broadcastToActors(createActivity, byActor, parentsCommentActors, actorsException)
|
||||
|
||||
// Broadcast to our followers
|
||||
await broadcastToFollowers(data, byActor, [ byActor ], t)
|
||||
await broadcastToFollowers(createActivity, byActor, [ byActor ], t)
|
||||
|
||||
// Send to actors involved in the comment
|
||||
if (isOrigin) return broadcastToFollowers(data, byActor, actorsInvolvedInComment, t, actorsException)
|
||||
if (isOrigin) return broadcastToFollowers(createActivity, byActor, actorsInvolvedInComment, t, actorsException)
|
||||
|
||||
// Send to origin
|
||||
return unicastTo(data, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
|
||||
return unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
|
||||
}
|
||||
|
||||
async function sendCreateView (byActor: ActorModel, video: VideoModel, t: Transaction) {
|
||||
logger.info('Creating job to send view of %s.', video.url)
|
||||
|
||||
const url = getVideoViewActivityPubUrl(byActor, video)
|
||||
const viewActivityData = createViewActivityData(byActor, video)
|
||||
const viewActivity = buildViewActivity(byActor, video)
|
||||
|
||||
const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
|
||||
|
||||
// Send to origin
|
||||
if (video.isOwned() === false) {
|
||||
const audience = getVideoAudience(video, actorsInvolvedInVideo)
|
||||
const data = createActivityData(url, byActor, viewActivityData, audience)
|
||||
const createActivity = buildCreateActivity(url, byActor, viewActivity, audience)
|
||||
|
||||
return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
|
||||
return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
|
||||
}
|
||||
|
||||
// Send to followers
|
||||
const audience = getObjectFollowersAudience(actorsInvolvedInVideo)
|
||||
const data = createActivityData(url, byActor, viewActivityData, audience)
|
||||
const createActivity = buildCreateActivity(url, byActor, viewActivity, audience)
|
||||
|
||||
// Use the server actor to send the view
|
||||
const serverActor = await getServerActor()
|
||||
const actorsException = [ byActor ]
|
||||
return broadcastToFollowers(data, serverActor, actorsInvolvedInVideo, t, actorsException)
|
||||
return broadcastToFollowers(createActivity, serverActor, actorsInvolvedInVideo, t, actorsException)
|
||||
}
|
||||
|
||||
async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Transaction) {
|
||||
logger.info('Creating job to dislike %s.', video.url)
|
||||
|
||||
const url = getVideoDislikeActivityPubUrl(byActor, video)
|
||||
const dislikeActivityData = createDislikeActivityData(byActor, video)
|
||||
const dislikeActivity = buildDislikeActivity(byActor, video)
|
||||
|
||||
const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
|
||||
|
||||
// Send to origin
|
||||
if (video.isOwned() === false) {
|
||||
const audience = getVideoAudience(video, actorsInvolvedInVideo)
|
||||
const data = createActivityData(url, byActor, dislikeActivityData, audience)
|
||||
const createActivity = buildCreateActivity(url, byActor, dislikeActivity, audience)
|
||||
|
||||
return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
|
||||
return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
|
||||
}
|
||||
|
||||
// Send to followers
|
||||
const audience = getObjectFollowersAudience(actorsInvolvedInVideo)
|
||||
const data = createActivityData(url, byActor, dislikeActivityData, audience)
|
||||
const createActivity = buildCreateActivity(url, byActor, dislikeActivity, audience)
|
||||
|
||||
const actorsException = [ byActor ]
|
||||
return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, actorsException)
|
||||
return broadcastToFollowers(createActivity, byActor, actorsInvolvedInVideo, t, actorsException)
|
||||
}
|
||||
|
||||
function createActivityData (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate {
|
||||
function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate {
|
||||
if (!audience) audience = getAudience(byActor)
|
||||
|
||||
return audiencify(
|
||||
@ -146,7 +161,7 @@ function createActivityData (url: string, byActor: ActorModel, object: any, audi
|
||||
)
|
||||
}
|
||||
|
||||
function createDislikeActivityData (byActor: ActorModel, video: VideoModel) {
|
||||
function buildDislikeActivity (byActor: ActorModel, video: VideoModel) {
|
||||
return {
|
||||
type: 'Dislike',
|
||||
actor: byActor.url,
|
||||
@ -154,7 +169,7 @@ function createDislikeActivityData (byActor: ActorModel, video: VideoModel) {
|
||||
}
|
||||
}
|
||||
|
||||
function createViewActivityData (byActor: ActorModel, video: VideoModel) {
|
||||
function buildViewActivity (byActor: ActorModel, video: VideoModel) {
|
||||
return {
|
||||
type: 'View',
|
||||
actor: byActor.url,
|
||||
@ -167,9 +182,10 @@ function createViewActivityData (byActor: ActorModel, video: VideoModel) {
|
||||
export {
|
||||
sendCreateVideo,
|
||||
sendVideoAbuse,
|
||||
createActivityData,
|
||||
buildCreateActivity,
|
||||
sendCreateView,
|
||||
sendCreateDislike,
|
||||
createDislikeActivityData,
|
||||
sendCreateVideoComment
|
||||
buildDislikeActivity,
|
||||
sendCreateVideoComment,
|
||||
sendCreateCacheFile
|
||||
}
|
||||
|
@ -15,24 +15,23 @@ async function sendDeleteVideo (video: VideoModel, t: Transaction) {
|
||||
const url = getDeleteActivityPubUrl(video.url)
|
||||
const byActor = video.VideoChannel.Account.Actor
|
||||
|
||||
const data = deleteActivityData(url, video.url, byActor)
|
||||
const activity = buildDeleteActivity(url, video.url, byActor)
|
||||
|
||||
const actorsInvolved = await VideoShareModel.loadActorsByShare(video.id, t)
|
||||
actorsInvolved.push(byActor)
|
||||
const actorsInvolved = await getActorsInvolvedInVideo(video, t)
|
||||
|
||||
return broadcastToFollowers(data, byActor, actorsInvolved, t)
|
||||
return broadcastToFollowers(activity, byActor, actorsInvolved, t)
|
||||
}
|
||||
|
||||
async function sendDeleteActor (byActor: ActorModel, t: Transaction) {
|
||||
logger.info('Creating job to broadcast delete of actor %s.', byActor.url)
|
||||
|
||||
const url = getDeleteActivityPubUrl(byActor.url)
|
||||
const data = deleteActivityData(url, byActor.url, byActor)
|
||||
const activity = buildDeleteActivity(url, byActor.url, byActor)
|
||||
|
||||
const actorsInvolved = await VideoShareModel.loadActorsByVideoOwner(byActor.id, t)
|
||||
actorsInvolved.push(byActor)
|
||||
|
||||
return broadcastToFollowers(data, byActor, actorsInvolved, t)
|
||||
return broadcastToFollowers(activity, byActor, actorsInvolved, t)
|
||||
}
|
||||
|
||||
async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Transaction) {
|
||||
@ -45,23 +44,23 @@ async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Trans
|
||||
const threadParentComments = await VideoCommentModel.listThreadParentComments(videoComment, t)
|
||||
|
||||
const actorsInvolvedInComment = await getActorsInvolvedInVideo(videoComment.Video, t)
|
||||
actorsInvolvedInComment.push(byActor)
|
||||
actorsInvolvedInComment.push(byActor) // Add the actor that commented the video
|
||||
|
||||
const audience = getVideoCommentAudience(videoComment, threadParentComments, actorsInvolvedInComment, isVideoOrigin)
|
||||
const data = deleteActivityData(url, videoComment.url, byActor, audience)
|
||||
const activity = buildDeleteActivity(url, videoComment.url, byActor, audience)
|
||||
|
||||
// This was a reply, send it to the parent actors
|
||||
const actorsException = [ byActor ]
|
||||
await broadcastToActors(data, byActor, threadParentComments.map(c => c.Account.Actor), actorsException)
|
||||
await broadcastToActors(activity, byActor, threadParentComments.map(c => c.Account.Actor), actorsException)
|
||||
|
||||
// Broadcast to our followers
|
||||
await broadcastToFollowers(data, byActor, [ byActor ], t)
|
||||
await broadcastToFollowers(activity, byActor, [ byActor ], t)
|
||||
|
||||
// Send to actors involved in the comment
|
||||
if (isVideoOrigin) return broadcastToFollowers(data, byActor, actorsInvolvedInComment, t, actorsException)
|
||||
if (isVideoOrigin) return broadcastToFollowers(activity, byActor, actorsInvolvedInComment, t, actorsException)
|
||||
|
||||
// Send to origin
|
||||
return unicastTo(data, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
|
||||
return unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@ -74,7 +73,7 @@ export {
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function deleteActivityData (url: string, object: string, byActor: ActorModel, audience?: ActivityAudience): ActivityDelete {
|
||||
function buildDeleteActivity (url: string, object: string, byActor: ActorModel, audience?: ActivityAudience): ActivityDelete {
|
||||
const activity = {
|
||||
type: 'Delete' as 'Delete',
|
||||
id: url,
|
||||
|
@ -15,12 +15,12 @@ function sendFollow (actorFollow: ActorFollowModel) {
|
||||
logger.info('Creating job to send follow request to %s.', following.url)
|
||||
|
||||
const url = getActorFollowActivityPubUrl(actorFollow)
|
||||
const data = followActivityData(url, me, following)
|
||||
const data = buildFollowActivity(url, me, following)
|
||||
|
||||
return unicastTo(data, me, following.inboxUrl)
|
||||
}
|
||||
|
||||
function followActivityData (url: string, byActor: ActorModel, targetActor: ActorModel): ActivityFollow {
|
||||
function buildFollowActivity (url: string, byActor: ActorModel, targetActor: ActorModel): ActivityFollow {
|
||||
return {
|
||||
type: 'Follow',
|
||||
id: url,
|
||||
@ -33,5 +33,5 @@ function followActivityData (url: string, byActor: ActorModel, targetActor: Acto
|
||||
|
||||
export {
|
||||
sendFollow,
|
||||
followActivityData
|
||||
buildFollowActivity
|
||||
}
|
||||
|
@ -17,20 +17,20 @@ async function sendLike (byActor: ActorModel, video: VideoModel, t: Transaction)
|
||||
// Send to origin
|
||||
if (video.isOwned() === false) {
|
||||
const audience = getVideoAudience(video, accountsInvolvedInVideo)
|
||||
const data = likeActivityData(url, byActor, video, audience)
|
||||
const data = buildLikeActivity(url, byActor, video, audience)
|
||||
|
||||
return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
|
||||
}
|
||||
|
||||
// Send to followers
|
||||
const audience = getObjectFollowersAudience(accountsInvolvedInVideo)
|
||||
const data = likeActivityData(url, byActor, video, audience)
|
||||
const activity = buildLikeActivity(url, byActor, video, audience)
|
||||
|
||||
const followersException = [ byActor ]
|
||||
return broadcastToFollowers(data, byActor, accountsInvolvedInVideo, t, followersException)
|
||||
return broadcastToFollowers(activity, byActor, accountsInvolvedInVideo, t, followersException)
|
||||
}
|
||||
|
||||
function likeActivityData (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityLike {
|
||||
function buildLikeActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityLike {
|
||||
if (!audience) audience = getAudience(byActor)
|
||||
|
||||
return audiencify(
|
||||
@ -48,5 +48,5 @@ function likeActivityData (url: string, byActor: ActorModel, video: VideoModel,
|
||||
|
||||
export {
|
||||
sendLike,
|
||||
likeActivityData
|
||||
buildLikeActivity
|
||||
}
|
||||
|
@ -13,12 +13,13 @@ import { VideoModel } from '../../../models/video/video'
|
||||
import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url'
|
||||
import { broadcastToFollowers, unicastTo } from './utils'
|
||||
import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience, getVideoAudience } from '../audience'
|
||||
import { createActivityData, createDislikeActivityData } from './send-create'
|
||||
import { followActivityData } from './send-follow'
|
||||
import { likeActivityData } from './send-like'
|
||||
import { buildCreateActivity, buildDislikeActivity } from './send-create'
|
||||
import { buildFollowActivity } from './send-follow'
|
||||
import { buildLikeActivity } from './send-like'
|
||||
import { VideoShareModel } from '../../../models/video/video-share'
|
||||
import { buildVideoAnnounce } from './send-announce'
|
||||
import { buildAnnounceWithVideoAudience } from './send-announce'
|
||||
import { logger } from '../../../helpers/logger'
|
||||
import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
|
||||
|
||||
async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) {
|
||||
const me = actorFollow.ActorFollower
|
||||
@ -32,10 +33,10 @@ async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) {
|
||||
const followUrl = getActorFollowActivityPubUrl(actorFollow)
|
||||
const undoUrl = getUndoActivityPubUrl(followUrl)
|
||||
|
||||
const object = followActivityData(followUrl, me, following)
|
||||
const data = undoActivityData(undoUrl, me, object)
|
||||
const followActivity = buildFollowActivity(followUrl, me, following)
|
||||
const undoActivity = undoActivityData(undoUrl, me, followActivity)
|
||||
|
||||
return unicastTo(data, me, following.inboxUrl)
|
||||
return unicastTo(undoActivity, me, following.inboxUrl)
|
||||
}
|
||||
|
||||
async function sendUndoLike (byActor: ActorModel, video: VideoModel, t: Transaction) {
|
||||
@ -45,21 +46,21 @@ async function sendUndoLike (byActor: ActorModel, video: VideoModel, t: Transact
|
||||
const undoUrl = getUndoActivityPubUrl(likeUrl)
|
||||
|
||||
const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
|
||||
const object = likeActivityData(likeUrl, byActor, video)
|
||||
const likeActivity = buildLikeActivity(likeUrl, byActor, video)
|
||||
|
||||
// Send to origin
|
||||
if (video.isOwned() === false) {
|
||||
const audience = getVideoAudience(video, actorsInvolvedInVideo)
|
||||
const data = undoActivityData(undoUrl, byActor, object, audience)
|
||||
const undoActivity = undoActivityData(undoUrl, byActor, likeActivity, audience)
|
||||
|
||||
return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
|
||||
return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
|
||||
}
|
||||
|
||||
const audience = getObjectFollowersAudience(actorsInvolvedInVideo)
|
||||
const data = undoActivityData(undoUrl, byActor, object, audience)
|
||||
const undoActivity = undoActivityData(undoUrl, byActor, likeActivity, audience)
|
||||
|
||||
const followersException = [ byActor ]
|
||||
return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, followersException)
|
||||
return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException)
|
||||
}
|
||||
|
||||
async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Transaction) {
|
||||
@ -69,20 +70,20 @@ async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Trans
|
||||
const undoUrl = getUndoActivityPubUrl(dislikeUrl)
|
||||
|
||||
const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
|
||||
const dislikeActivity = createDislikeActivityData(byActor, video)
|
||||
const object = createActivityData(dislikeUrl, byActor, dislikeActivity)
|
||||
const dislikeActivity = buildDislikeActivity(byActor, video)
|
||||
const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity)
|
||||
|
||||
if (video.isOwned() === false) {
|
||||
const audience = getVideoAudience(video, actorsInvolvedInVideo)
|
||||
const data = undoActivityData(undoUrl, byActor, object, audience)
|
||||
const undoActivity = undoActivityData(undoUrl, byActor, createDislikeActivity, audience)
|
||||
|
||||
return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
|
||||
return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
|
||||
}
|
||||
|
||||
const data = undoActivityData(undoUrl, byActor, object)
|
||||
const undoActivity = undoActivityData(undoUrl, byActor, createDislikeActivity)
|
||||
|
||||
const followersException = [ byActor ]
|
||||
return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, followersException)
|
||||
return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException)
|
||||
}
|
||||
|
||||
async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
|
||||
@ -90,12 +91,27 @@ async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareMode
|
||||
|
||||
const undoUrl = getUndoActivityPubUrl(videoShare.url)
|
||||
|
||||
const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
|
||||
const object = await buildVideoAnnounce(byActor, videoShare, video, t)
|
||||
const data = undoActivityData(undoUrl, byActor, object)
|
||||
const { activity: announceActivity, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, t)
|
||||
const undoActivity = undoActivityData(undoUrl, byActor, announceActivity)
|
||||
|
||||
const followersException = [ byActor ]
|
||||
return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, followersException)
|
||||
return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException)
|
||||
}
|
||||
|
||||
async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) {
|
||||
logger.info('Creating job to undo cache file %s.', redundancyModel.url)
|
||||
|
||||
const undoUrl = getUndoActivityPubUrl(redundancyModel.url)
|
||||
|
||||
const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id)
|
||||
const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
|
||||
|
||||
const audience = getVideoAudience(video, actorsInvolvedInVideo)
|
||||
const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject())
|
||||
|
||||
const undoActivity = undoActivityData(undoUrl, byActor, createActivity, audience)
|
||||
|
||||
return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@ -104,7 +120,8 @@ export {
|
||||
sendUndoFollow,
|
||||
sendUndoLike,
|
||||
sendUndoDislike,
|
||||
sendUndoAnnounce
|
||||
sendUndoAnnounce,
|
||||
sendUndoCacheFile
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
@ -7,11 +7,11 @@ import { VideoModel } from '../../../models/video/video'
|
||||
import { VideoChannelModel } from '../../../models/video/video-channel'
|
||||
import { VideoShareModel } from '../../../models/video/video-share'
|
||||
import { getUpdateActivityPubUrl } from '../url'
|
||||
import { broadcastToFollowers } from './utils'
|
||||
import { audiencify, getAudience } from '../audience'
|
||||
import { broadcastToFollowers, unicastTo } from './utils'
|
||||
import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience'
|
||||
import { logger } from '../../../helpers/logger'
|
||||
import { videoFeedsValidator } from '../../../middlewares/validators'
|
||||
import { VideoCaptionModel } from '../../../models/video/video-caption'
|
||||
import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
|
||||
|
||||
async function sendUpdateVideo (video: VideoModel, t: Transaction, overrodeByActor?: ActorModel) {
|
||||
logger.info('Creating job to update video %s.', video.url)
|
||||
@ -26,12 +26,12 @@ async function sendUpdateVideo (video: VideoModel, t: Transaction, overrodeByAct
|
||||
const videoObject = video.toActivityPubObject()
|
||||
const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC)
|
||||
|
||||
const data = updateActivityData(url, byActor, videoObject, audience)
|
||||
const updateActivity = buildUpdateActivity(url, byActor, videoObject, audience)
|
||||
|
||||
const actorsInvolved = await VideoShareModel.loadActorsByShare(video.id, t)
|
||||
actorsInvolved.push(byActor)
|
||||
const actorsInvolved = await getActorsInvolvedInVideo(video, t)
|
||||
if (overrodeByActor) actorsInvolved.push(overrodeByActor)
|
||||
|
||||
return broadcastToFollowers(data, byActor, actorsInvolved, t)
|
||||
return broadcastToFollowers(updateActivity, byActor, actorsInvolved, t)
|
||||
}
|
||||
|
||||
async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelModel, t: Transaction) {
|
||||
@ -42,7 +42,7 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod
|
||||
const url = getUpdateActivityPubUrl(byActor.url, byActor.updatedAt.toISOString())
|
||||
const accountOrChannelObject = accountOrChannel.toActivityPubObject()
|
||||
const audience = getAudience(byActor)
|
||||
const data = updateActivityData(url, byActor, accountOrChannelObject, audience)
|
||||
const updateActivity = buildUpdateActivity(url, byActor, accountOrChannelObject, audience)
|
||||
|
||||
let actorsInvolved: ActorModel[]
|
||||
if (accountOrChannel instanceof AccountModel) {
|
||||
@ -55,19 +55,35 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod
|
||||
|
||||
actorsInvolved.push(byActor)
|
||||
|
||||
return broadcastToFollowers(data, byActor, actorsInvolved, t)
|
||||
return broadcastToFollowers(updateActivity, byActor, actorsInvolved, t)
|
||||
}
|
||||
|
||||
async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) {
|
||||
logger.info('Creating job to update cache file %s.', redundancyModel.url)
|
||||
|
||||
const url = getUpdateActivityPubUrl(redundancyModel.url, redundancyModel.updatedAt.toISOString())
|
||||
const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id)
|
||||
|
||||
const redundancyObject = redundancyModel.toActivityPubObject()
|
||||
|
||||
const accountsInvolvedInVideo = await getActorsInvolvedInVideo(video, undefined)
|
||||
const audience = getObjectFollowersAudience(accountsInvolvedInVideo)
|
||||
|
||||
const updateActivity = buildUpdateActivity(url, byActor, redundancyObject, audience)
|
||||
return unicastTo(updateActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
sendUpdateActor,
|
||||
sendUpdateVideo
|
||||
sendUpdateVideo,
|
||||
sendUpdateCacheFile
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function updateActivityData (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityUpdate {
|
||||
function buildUpdateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityUpdate {
|
||||
if (!audience) audience = getAudience(byActor)
|
||||
|
||||
return audiencify(
|
||||
|
@ -59,11 +59,11 @@ async function forwardActivity (
|
||||
async function broadcastToFollowers (
|
||||
data: any,
|
||||
byActor: ActorModel,
|
||||
toActorFollowers: ActorModel[],
|
||||
toFollowersOf: ActorModel[],
|
||||
t: Transaction,
|
||||
actorsException: ActorModel[] = []
|
||||
) {
|
||||
const uris = await computeFollowerUris(toActorFollowers, actorsException, t)
|
||||
const uris = await computeFollowerUris(toFollowersOf, actorsException, t)
|
||||
return broadcastTo(uris, data, byActor)
|
||||
}
|
||||
|
||||
@ -115,8 +115,8 @@ export {
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function computeFollowerUris (toActorFollower: ActorModel[], actorsException: ActorModel[], t: Transaction) {
|
||||
const toActorFollowerIds = toActorFollower.map(a => a.id)
|
||||
async function computeFollowerUris (toFollowersOf: ActorModel[], actorsException: ActorModel[], t: Transaction) {
|
||||
const toActorFollowerIds = toFollowersOf.map(a => a.id)
|
||||
|
||||
const result = await ActorFollowModel.listAcceptedFollowerSharedInboxUrls(toActorFollowerIds, t)
|
||||
const sharedInboxesException = await buildSharedInboxesException(actorsException)
|
||||
|
@ -4,11 +4,18 @@ import { ActorFollowModel } from '../../models/activitypub/actor-follow'
|
||||
import { VideoModel } from '../../models/video/video'
|
||||
import { VideoAbuseModel } from '../../models/video/video-abuse'
|
||||
import { VideoCommentModel } from '../../models/video/video-comment'
|
||||
import { VideoFileModel } from '../../models/video/video-file'
|
||||
|
||||
function getVideoActivityPubUrl (video: VideoModel) {
|
||||
return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid
|
||||
}
|
||||
|
||||
function getVideoCacheFileActivityPubUrl (videoFile: VideoFileModel) {
|
||||
const suffixFPS = videoFile.fps ? '-' + videoFile.fps : ''
|
||||
|
||||
return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}`
|
||||
}
|
||||
|
||||
function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) {
|
||||
return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id
|
||||
}
|
||||
@ -101,5 +108,6 @@ export {
|
||||
getVideoSharesActivityPubUrl,
|
||||
getVideoCommentsActivityPubUrl,
|
||||
getVideoLikesActivityPubUrl,
|
||||
getVideoDislikesActivityPubUrl
|
||||
getVideoDislikesActivityPubUrl,
|
||||
getVideoCacheFileActivityPubUrl
|
||||
}
|
||||
|
@ -3,12 +3,12 @@ import * as sequelize from 'sequelize'
|
||||
import * as magnetUtil from 'magnet-uri'
|
||||
import { join } from 'path'
|
||||
import * as request from 'request'
|
||||
import { ActivityIconObject, VideoState } from '../../../shared/index'
|
||||
import { ActivityIconObject, ActivityVideoUrlObject, VideoState, ActivityUrlObject } from '../../../shared/index'
|
||||
import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
|
||||
import { VideoPrivacy } from '../../../shared/models/videos'
|
||||
import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
|
||||
import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
|
||||
import { resetSequelizeInstance, retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
|
||||
import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
|
||||
import { logger } from '../../helpers/logger'
|
||||
import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
|
||||
import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers'
|
||||
@ -17,7 +17,7 @@ import { TagModel } from '../../models/video/tag'
|
||||
import { VideoModel } from '../../models/video/video'
|
||||
import { VideoChannelModel } from '../../models/video/video-channel'
|
||||
import { VideoFileModel } from '../../models/video/video-file'
|
||||
import { getOrCreateActorAndServerAndModel, updateActorAvatarInstance } from './actor'
|
||||
import { getOrCreateActorAndServerAndModel } from './actor'
|
||||
import { addVideoComments } from './video-comments'
|
||||
import { crawlCollectionPage } from './crawl'
|
||||
import { sendCreateVideo, sendUpdateVideo } from './send'
|
||||
@ -25,7 +25,6 @@ import { isArray } from '../../helpers/custom-validators/misc'
|
||||
import { VideoCaptionModel } from '../../models/video/video-caption'
|
||||
import { JobQueue } from '../job-queue'
|
||||
import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher'
|
||||
import { getUrlFromWebfinger } from '../../helpers/webfinger'
|
||||
import { createRates } from './video-rates'
|
||||
import { addVideoShares, shareVideoByServerAndChannel } from './share'
|
||||
import { AccountModel } from '../../models/account/account'
|
||||
@ -137,10 +136,7 @@ async function videoActivityObjectToDBAttributes (
|
||||
}
|
||||
|
||||
function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) {
|
||||
const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT)
|
||||
const fileUrls = videoObject.url.filter(u => {
|
||||
return mimeTypes.indexOf(u.mimeType) !== -1 && u.mimeType.startsWith('video/')
|
||||
})
|
||||
const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[]
|
||||
|
||||
if (fileUrls.length === 0) {
|
||||
throw new Error('Cannot find video files for ' + videoCreated.url)
|
||||
@ -331,8 +327,8 @@ async function refreshVideoIfNeeded (video: VideoModel): Promise<VideoModel> {
|
||||
|
||||
const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
|
||||
const account = await AccountModel.load(channelActor.VideoChannel.accountId)
|
||||
return updateVideoFromAP(video, videoObject, account.Actor, channelActor)
|
||||
|
||||
return updateVideoFromAP(video, videoObject, account, channelActor.VideoChannel)
|
||||
} catch (err) {
|
||||
logger.warn('Cannot refresh video.', { err })
|
||||
return video
|
||||
@ -342,8 +338,8 @@ async function refreshVideoIfNeeded (video: VideoModel): Promise<VideoModel> {
|
||||
async function updateVideoFromAP (
|
||||
video: VideoModel,
|
||||
videoObject: VideoTorrentObject,
|
||||
accountActor: ActorModel,
|
||||
channelActor: ActorModel,
|
||||
account: AccountModel,
|
||||
channel: VideoChannelModel,
|
||||
overrideTo?: string[]
|
||||
) {
|
||||
logger.debug('Updating remote video "%s".', videoObject.uuid)
|
||||
@ -359,12 +355,12 @@ async function updateVideoFromAP (
|
||||
|
||||
// Check actor has the right to update the video
|
||||
const videoChannel = video.VideoChannel
|
||||
if (videoChannel.Account.Actor.id !== accountActor.id) {
|
||||
throw new Error('Account ' + accountActor.url + ' does not own video channel ' + videoChannel.Actor.url)
|
||||
if (videoChannel.Account.id !== account.id) {
|
||||
throw new Error('Account ' + account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url)
|
||||
}
|
||||
|
||||
const to = overrideTo ? overrideTo : videoObject.to
|
||||
const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, to)
|
||||
const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, to)
|
||||
video.set('name', videoData.name)
|
||||
video.set('uuid', videoData.uuid)
|
||||
video.set('url', videoData.url)
|
||||
@ -444,3 +440,11 @@ export {
|
||||
addVideoShares,
|
||||
createRates
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject {
|
||||
const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT)
|
||||
|
||||
return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/')
|
||||
}
|
||||
|
18
server/lib/redundancy.ts
Normal file
18
server/lib/redundancy.ts
Normal file
@ -0,0 +1,18 @@
|
||||
import { VideoRedundancyModel } from '../models/redundancy/video-redundancy'
|
||||
import { sendUndoCacheFile } from './activitypub/send'
|
||||
import { Transaction } from 'sequelize'
|
||||
import { getServerActor } from '../helpers/utils'
|
||||
|
||||
async function removeVideoRedundancy (videoRedundancy: VideoRedundancyModel, t?: Transaction) {
|
||||
const serverActor = await getServerActor()
|
||||
|
||||
await sendUndoCacheFile(serverActor, videoRedundancy, t)
|
||||
|
||||
await videoRedundancy.destroy({ transaction: t })
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
removeVideoRedundancy
|
||||
}
|
161
server/lib/schedulers/videos-redundancy-scheduler.ts
Normal file
161
server/lib/schedulers/videos-redundancy-scheduler.ts
Normal file
@ -0,0 +1,161 @@
|
||||
import { AbstractScheduler } from './abstract-scheduler'
|
||||
import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers'
|
||||
import { logger } from '../../helpers/logger'
|
||||
import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
|
||||
import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
|
||||
import { VideoFileModel } from '../../models/video/video-file'
|
||||
import { sortBy } from 'lodash'
|
||||
import { downloadWebTorrentVideo } from '../../helpers/webtorrent'
|
||||
import { join } from 'path'
|
||||
import { rename } from 'fs-extra'
|
||||
import { getServerActor } from '../../helpers/utils'
|
||||
import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
|
||||
import { VideoModel } from '../../models/video/video'
|
||||
import { getVideoCacheFileActivityPubUrl } from '../activitypub/url'
|
||||
import { removeVideoRedundancy } from '../redundancy'
|
||||
import { isTestInstance } from '../../helpers/core-utils'
|
||||
|
||||
export class VideosRedundancyScheduler extends AbstractScheduler {
|
||||
|
||||
private static instance: AbstractScheduler
|
||||
private executing = false
|
||||
|
||||
protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.videosRedundancy
|
||||
|
||||
private constructor () {
|
||||
super()
|
||||
}
|
||||
|
||||
async execute () {
|
||||
if (this.executing) return
|
||||
|
||||
this.executing = true
|
||||
|
||||
for (const obj of CONFIG.REDUNDANCY.VIDEOS) {
|
||||
|
||||
try {
|
||||
const videoToDuplicate = await this.findVideoToDuplicate(obj.strategy)
|
||||
if (!videoToDuplicate) continue
|
||||
|
||||
const videoFiles = videoToDuplicate.VideoFiles
|
||||
videoFiles.forEach(f => f.Video = videoToDuplicate)
|
||||
|
||||
const videosRedundancy = await VideoRedundancyModel.getVideoFiles(obj.strategy)
|
||||
if (this.isTooHeavy(videosRedundancy, videoFiles, obj.size)) {
|
||||
if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url)
|
||||
continue
|
||||
}
|
||||
|
||||
logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy)
|
||||
|
||||
await this.createVideoRedundancy(obj.strategy, videoFiles)
|
||||
} catch (err) {
|
||||
logger.error('Cannot run videos redundancy %s.', obj.strategy, { err })
|
||||
}
|
||||
}
|
||||
|
||||
const expired = await VideoRedundancyModel.listAllExpired()
|
||||
|
||||
for (const m of expired) {
|
||||
logger.info('Removing expired video %s from our redundancy system.', this.buildEntryLogId(m))
|
||||
|
||||
try {
|
||||
await m.destroy()
|
||||
} catch (err) {
|
||||
logger.error('Cannot remove %s video from our redundancy system.', this.buildEntryLogId(m))
|
||||
}
|
||||
}
|
||||
|
||||
this.executing = false
|
||||
}
|
||||
|
||||
static get Instance () {
|
||||
return this.instance || (this.instance = new this())
|
||||
}
|
||||
|
||||
private findVideoToDuplicate (strategy: VideoRedundancyStrategy) {
|
||||
if (strategy === 'most-views') return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
|
||||
}
|
||||
|
||||
private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) {
|
||||
const serverActor = await getServerActor()
|
||||
|
||||
for (const file of filesToDuplicate) {
|
||||
const existing = await VideoRedundancyModel.loadByFileId(file.id)
|
||||
if (existing) {
|
||||
logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', file.Video.url, file.resolution, strategy)
|
||||
|
||||
existing.expiresOn = this.buildNewExpiration()
|
||||
await existing.save()
|
||||
|
||||
await sendUpdateCacheFile(serverActor, existing)
|
||||
continue
|
||||
}
|
||||
|
||||
// We need more attributes and check if the video still exists
|
||||
const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(file.Video.id)
|
||||
if (!video) continue
|
||||
|
||||
logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy)
|
||||
|
||||
const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
|
||||
const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs)
|
||||
|
||||
const tmpPath = await downloadWebTorrentVideo({ magnetUri }, JOB_TTL['video-import'])
|
||||
|
||||
const destPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(file))
|
||||
await rename(tmpPath, destPath)
|
||||
|
||||
const createdModel = await VideoRedundancyModel.create({
|
||||
expiresOn: new Date(Date.now() + REDUNDANCY.VIDEOS.EXPIRES_AFTER_MS),
|
||||
url: getVideoCacheFileActivityPubUrl(file),
|
||||
fileUrl: video.getVideoFileUrl(file, CONFIG.WEBSERVER.URL),
|
||||
strategy,
|
||||
videoFileId: file.id,
|
||||
actorId: serverActor.id
|
||||
})
|
||||
createdModel.VideoFile = file
|
||||
|
||||
await sendCreateCacheFile(serverActor, createdModel)
|
||||
}
|
||||
}
|
||||
|
||||
// Unused, but could be useful in the future, with a custom strategy
|
||||
private async purgeVideosIfNeeded (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSize: number) {
|
||||
const sortedVideosRedundancy = sortBy(videosRedundancy, 'createdAt')
|
||||
|
||||
while (this.isTooHeavy(sortedVideosRedundancy, filesToDuplicate, maxSize)) {
|
||||
const toDelete = sortedVideosRedundancy.shift()
|
||||
|
||||
const videoFile = toDelete.VideoFile
|
||||
logger.info('Purging video %s (resolution %d) from our redundancy system.', videoFile.Video.url, videoFile.resolution)
|
||||
|
||||
await removeVideoRedundancy(toDelete, undefined)
|
||||
}
|
||||
|
||||
return sortedVideosRedundancy
|
||||
}
|
||||
|
||||
private isTooHeavy (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSizeArg: number) {
|
||||
const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate)
|
||||
|
||||
const redundancyReducer = (previous: number, current: VideoRedundancyModel) => previous + current.VideoFile.size
|
||||
const totalDuplicated = videosRedundancy.reduce(redundancyReducer, 0)
|
||||
|
||||
return totalDuplicated > maxSize
|
||||
}
|
||||
|
||||
private buildNewExpiration () {
|
||||
return new Date(Date.now() + REDUNDANCY.VIDEOS.EXPIRES_AFTER_MS)
|
||||
}
|
||||
|
||||
private buildEntryLogId (object: VideoRedundancyModel) {
|
||||
return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}`
|
||||
}
|
||||
|
||||
private getTotalFileSizes (files: VideoFileModel[]) {
|
||||
const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size
|
||||
|
||||
return files.reduce(fileReducer, 0)
|
||||
}
|
||||
}
|
80
server/middlewares/validators/redundancy.ts
Normal file
80
server/middlewares/validators/redundancy.ts
Normal file
@ -0,0 +1,80 @@
|
||||
import * as express from 'express'
|
||||
import 'express-validator'
|
||||
import { param, body } from 'express-validator/check'
|
||||
import { exists, isBooleanValid, isIdOrUUIDValid, toIntOrNull } from '../../helpers/custom-validators/misc'
|
||||
import { isVideoExist } from '../../helpers/custom-validators/videos'
|
||||
import { logger } from '../../helpers/logger'
|
||||
import { areValidationErrors } from './utils'
|
||||
import { VideoModel } from '../../models/video/video'
|
||||
import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
|
||||
import { isHostValid } from '../../helpers/custom-validators/servers'
|
||||
import { getServerActor } from '../../helpers/utils'
|
||||
import { ActorFollowModel } from '../../models/activitypub/actor-follow'
|
||||
import { SERVER_ACTOR_NAME } from '../../initializers'
|
||||
import { ServerModel } from '../../models/server/server'
|
||||
|
||||
const videoRedundancyGetValidator = [
|
||||
param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'),
|
||||
param('resolution')
|
||||
.customSanitizer(toIntOrNull)
|
||||
.custom(exists).withMessage('Should have a valid resolution'),
|
||||
param('fps')
|
||||
.optional()
|
||||
.customSanitizer(toIntOrNull)
|
||||
.custom(exists).withMessage('Should have a valid fps'),
|
||||
|
||||
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
|
||||
logger.debug('Checking videoRedundancyGetValidator parameters', { parameters: req.params })
|
||||
|
||||
if (areValidationErrors(req, res)) return
|
||||
if (!await isVideoExist(req.params.videoId, res)) return
|
||||
|
||||
const video: VideoModel = res.locals.video
|
||||
const videoFile = video.VideoFiles.find(f => {
|
||||
return f.resolution === req.params.resolution && (!req.params.fps || f.fps === req.params.fps)
|
||||
})
|
||||
|
||||
if (!videoFile) return res.status(404).json({ error: 'Video file not found.' })
|
||||
res.locals.videoFile = videoFile
|
||||
|
||||
const videoRedundancy = await VideoRedundancyModel.loadByFileId(videoFile.id)
|
||||
if (!videoRedundancy)return res.status(404).json({ error: 'Video redundancy not found.' })
|
||||
res.locals.videoRedundancy = videoRedundancy
|
||||
|
||||
return next()
|
||||
}
|
||||
]
|
||||
|
||||
const updateServerRedundancyValidator = [
|
||||
param('host').custom(isHostValid).withMessage('Should have a valid host'),
|
||||
body('redundancyAllowed')
|
||||
.toBoolean()
|
||||
.custom(isBooleanValid).withMessage('Should have a valid redundancyAllowed attribute'),
|
||||
|
||||
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
|
||||
logger.debug('Checking updateServerRedundancy parameters', { parameters: req.params })
|
||||
|
||||
if (areValidationErrors(req, res)) return
|
||||
|
||||
const server = await ServerModel.loadByHost(req.params.host)
|
||||
|
||||
if (!server) {
|
||||
return res
|
||||
.status(404)
|
||||
.json({
|
||||
error: `Server ${req.params.host} not found.`
|
||||
})
|
||||
.end()
|
||||
}
|
||||
|
||||
res.locals.server = server
|
||||
return next()
|
||||
}
|
||||
]
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
videoRedundancyGetValidator,
|
||||
updateServerRedundancyValidator
|
||||
}
|
@ -19,7 +19,7 @@ import {
|
||||
UpdatedAt
|
||||
} from 'sequelize-typescript'
|
||||
import { FollowState } from '../../../shared/models/actors'
|
||||
import { AccountFollow } from '../../../shared/models/actors/follow.model'
|
||||
import { ActorFollow } from '../../../shared/models/actors/follow.model'
|
||||
import { logger } from '../../helpers/logger'
|
||||
import { getServerActor } from '../../helpers/utils'
|
||||
import { ACTOR_FOLLOW_SCORE } from '../../initializers'
|
||||
@ -529,7 +529,7 @@ export class ActorFollowModel extends Model<ActorFollowModel> {
|
||||
return ActorFollowModel.findAll(query)
|
||||
}
|
||||
|
||||
toFormattedJSON (): AccountFollow {
|
||||
toFormattedJSON (): ActorFollow {
|
||||
const follower = this.ActorFollower.toFormattedJSON()
|
||||
const following = this.ActorFollowing.toFormattedJSON()
|
||||
|
||||
|
@ -76,7 +76,13 @@ export const unusedActorAttributesForAPI = [
|
||||
},
|
||||
{
|
||||
model: () => VideoChannelModel.unscoped(),
|
||||
required: false
|
||||
required: false,
|
||||
include: [
|
||||
{
|
||||
model: () => AccountModel,
|
||||
required: true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
model: () => ServerModel,
|
||||
@ -337,6 +343,7 @@ export class ActorModel extends Model<ActorModel> {
|
||||
uuid: this.uuid,
|
||||
name: this.preferredUsername,
|
||||
host: this.getHost(),
|
||||
hostRedundancyAllowed: this.getRedundancyAllowed(),
|
||||
followingCount: this.followingCount,
|
||||
followersCount: this.followersCount,
|
||||
avatar,
|
||||
@ -440,6 +447,10 @@ export class ActorModel extends Model<ActorModel> {
|
||||
return this.Server ? this.Server.host : CONFIG.WEBSERVER.HOST
|
||||
}
|
||||
|
||||
getRedundancyAllowed () {
|
||||
return this.Server ? this.Server.redundancyAllowed : false
|
||||
}
|
||||
|
||||
getAvatarUrl () {
|
||||
if (!this.avatarId) return undefined
|
||||
|
||||
|
249
server/models/redundancy/video-redundancy.ts
Normal file
249
server/models/redundancy/video-redundancy.ts
Normal file
@ -0,0 +1,249 @@
|
||||
import {
|
||||
AfterDestroy,
|
||||
AllowNull,
|
||||
BelongsTo,
|
||||
Column,
|
||||
CreatedAt,
|
||||
DataType,
|
||||
ForeignKey,
|
||||
Is,
|
||||
Model,
|
||||
Scopes,
|
||||
Sequelize,
|
||||
Table,
|
||||
UpdatedAt
|
||||
} from 'sequelize-typescript'
|
||||
import { ActorModel } from '../activitypub/actor'
|
||||
import { throwIfNotValid } from '../utils'
|
||||
import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
|
||||
import { CONSTRAINTS_FIELDS, VIDEO_EXT_MIMETYPE } from '../../initializers'
|
||||
import { VideoFileModel } from '../video/video-file'
|
||||
import { isDateValid } from '../../helpers/custom-validators/misc'
|
||||
import { getServerActor } from '../../helpers/utils'
|
||||
import { VideoModel } from '../video/video'
|
||||
import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
|
||||
import { logger } from '../../helpers/logger'
|
||||
import { CacheFileObject } from '../../../shared'
|
||||
import { VideoChannelModel } from '../video/video-channel'
|
||||
import { ServerModel } from '../server/server'
|
||||
import { sample } from 'lodash'
|
||||
import { isTestInstance } from '../../helpers/core-utils'
|
||||
|
||||
export enum ScopeNames {
|
||||
WITH_VIDEO = 'WITH_VIDEO'
|
||||
}
|
||||
|
||||
@Scopes({
|
||||
[ ScopeNames.WITH_VIDEO ]: {
|
||||
include: [
|
||||
{
|
||||
model: () => VideoFileModel,
|
||||
required: true,
|
||||
include: [
|
||||
{
|
||||
model: () => VideoModel,
|
||||
required: true
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
})
|
||||
|
||||
@Table({
|
||||
tableName: 'videoRedundancy',
|
||||
indexes: [
|
||||
{
|
||||
fields: [ 'videoFileId' ]
|
||||
},
|
||||
{
|
||||
fields: [ 'actorId' ]
|
||||
},
|
||||
{
|
||||
fields: [ 'url' ],
|
||||
unique: true
|
||||
}
|
||||
]
|
||||
})
|
||||
export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
|
||||
|
||||
@CreatedAt
|
||||
createdAt: Date
|
||||
|
||||
@UpdatedAt
|
||||
updatedAt: Date
|
||||
|
||||
@AllowNull(false)
|
||||
@Column
|
||||
expiresOn: Date
|
||||
|
||||
@AllowNull(false)
|
||||
@Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
|
||||
@Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
|
||||
fileUrl: string
|
||||
|
||||
@AllowNull(false)
|
||||
@Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
|
||||
@Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
|
||||
url: string
|
||||
|
||||
@AllowNull(true)
|
||||
@Column
|
||||
strategy: string // Only used by us
|
||||
|
||||
@ForeignKey(() => VideoFileModel)
|
||||
@Column
|
||||
videoFileId: number
|
||||
|
||||
@BelongsTo(() => VideoFileModel, {
|
||||
foreignKey: {
|
||||
allowNull: false
|
||||
},
|
||||
onDelete: 'cascade'
|
||||
})
|
||||
VideoFile: VideoFileModel
|
||||
|
||||
@ForeignKey(() => ActorModel)
|
||||
@Column
|
||||
actorId: number
|
||||
|
||||
@BelongsTo(() => ActorModel, {
|
||||
foreignKey: {
|
||||
allowNull: false
|
||||
},
|
||||
onDelete: 'cascade'
|
||||
})
|
||||
Actor: ActorModel
|
||||
|
||||
@AfterDestroy
|
||||
static removeFilesAndSendDelete (instance: VideoRedundancyModel) {
|
||||
// Not us
|
||||
if (!instance.strategy) return
|
||||
|
||||
logger.info('Removing video file %s-.', instance.VideoFile.Video.uuid, instance.VideoFile.resolution)
|
||||
|
||||
return instance.VideoFile.Video.removeFile(instance.VideoFile)
|
||||
}
|
||||
|
||||
static loadByFileId (videoFileId: number) {
|
||||
const query = {
|
||||
where: {
|
||||
videoFileId
|
||||
}
|
||||
}
|
||||
|
||||
return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
|
||||
}
|
||||
|
||||
static loadByUrl (url: string) {
|
||||
const query = {
|
||||
where: {
|
||||
url
|
||||
}
|
||||
}
|
||||
|
||||
return VideoRedundancyModel.findOne(query)
|
||||
}
|
||||
|
||||
static async findMostViewToDuplicate (randomizedFactor: number) {
|
||||
// On VideoModel!
|
||||
const query = {
|
||||
logging: !isTestInstance(),
|
||||
limit: randomizedFactor,
|
||||
order: [ [ 'views', 'DESC' ] ],
|
||||
include: [
|
||||
{
|
||||
model: VideoFileModel.unscoped(),
|
||||
required: true,
|
||||
where: {
|
||||
id: {
|
||||
[ Sequelize.Op.notIn ]: await VideoRedundancyModel.buildExcludeIn()
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
attributes: [],
|
||||
model: VideoChannelModel.unscoped(),
|
||||
required: true,
|
||||
include: [
|
||||
{
|
||||
attributes: [],
|
||||
model: ActorModel.unscoped(),
|
||||
required: true,
|
||||
include: [
|
||||
{
|
||||
attributes: [],
|
||||
model: ServerModel.unscoped(),
|
||||
required: true,
|
||||
where: {
|
||||
redundancyAllowed: true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
const rows = await VideoModel.unscoped().findAll(query)
|
||||
|
||||
return sample(rows)
|
||||
}
|
||||
|
||||
static async getVideoFiles (strategy: VideoRedundancyStrategy) {
|
||||
const actor = await getServerActor()
|
||||
|
||||
const queryVideoFiles = {
|
||||
logging: !isTestInstance(),
|
||||
where: {
|
||||
actorId: actor.id,
|
||||
strategy
|
||||
}
|
||||
}
|
||||
|
||||
return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO)
|
||||
.findAll(queryVideoFiles)
|
||||
}
|
||||
|
||||
static listAllExpired () {
|
||||
const query = {
|
||||
logging: !isTestInstance(),
|
||||
where: {
|
||||
expiresOn: {
|
||||
[Sequelize.Op.lt]: new Date()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO)
|
||||
.findAll(query)
|
||||
}
|
||||
|
||||
toActivityPubObject (): CacheFileObject {
|
||||
return {
|
||||
id: this.url,
|
||||
type: 'CacheFile' as 'CacheFile',
|
||||
object: this.VideoFile.Video.url,
|
||||
expires: this.expiresOn.toISOString(),
|
||||
url: {
|
||||
type: 'Link',
|
||||
mimeType: VIDEO_EXT_MIMETYPE[ this.VideoFile.extname ] as any,
|
||||
href: this.fileUrl,
|
||||
height: this.VideoFile.resolution,
|
||||
size: this.VideoFile.size,
|
||||
fps: this.VideoFile.fps
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static async buildExcludeIn () {
|
||||
const actor = await getServerActor()
|
||||
|
||||
return Sequelize.literal(
|
||||
'(' +
|
||||
`SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "expiresOn" >= NOW()` +
|
||||
')'
|
||||
)
|
||||
}
|
||||
}
|
@ -1,4 +1,4 @@
|
||||
import { AllowNull, Column, CreatedAt, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
|
||||
import { AllowNull, Column, CreatedAt, Default, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
|
||||
import { isHostValid } from '../../helpers/custom-validators/servers'
|
||||
import { ActorModel } from '../activitypub/actor'
|
||||
import { throwIfNotValid } from '../utils'
|
||||
@ -19,6 +19,11 @@ export class ServerModel extends Model<ServerModel> {
|
||||
@Column
|
||||
host: string
|
||||
|
||||
@AllowNull(false)
|
||||
@Default(false)
|
||||
@Column
|
||||
redundancyAllowed: boolean
|
||||
|
||||
@CreatedAt
|
||||
createdAt: Date
|
||||
|
||||
@ -34,4 +39,14 @@ export class ServerModel extends Model<ServerModel> {
|
||||
hooks: true
|
||||
})
|
||||
Actors: ActorModel[]
|
||||
|
||||
static loadByHost (host: string) {
|
||||
const query = {
|
||||
where: {
|
||||
host
|
||||
}
|
||||
}
|
||||
|
||||
return ServerModel.findOne(query)
|
||||
}
|
||||
}
|
||||
|
@ -1,5 +1,18 @@
|
||||
import { values } from 'lodash'
|
||||
import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
|
||||
import {
|
||||
AllowNull,
|
||||
BelongsTo,
|
||||
Column,
|
||||
CreatedAt,
|
||||
DataType,
|
||||
Default,
|
||||
ForeignKey,
|
||||
HasMany,
|
||||
Is,
|
||||
Model,
|
||||
Table,
|
||||
UpdatedAt
|
||||
} from 'sequelize-typescript'
|
||||
import {
|
||||
isVideoFileInfoHashValid,
|
||||
isVideoFileResolutionValid,
|
||||
@ -10,6 +23,7 @@ import { CONSTRAINTS_FIELDS } from '../../initializers'
|
||||
import { throwIfNotValid } from '../utils'
|
||||
import { VideoModel } from './video'
|
||||
import * as Sequelize from 'sequelize'
|
||||
import { VideoRedundancyModel } from '../redundancy/video-redundancy'
|
||||
|
||||
@Table({
|
||||
tableName: 'videoFile',
|
||||
@ -70,6 +84,15 @@ export class VideoFileModel extends Model<VideoFileModel> {
|
||||
})
|
||||
Video: VideoModel
|
||||
|
||||
@HasMany(() => VideoRedundancyModel, {
|
||||
foreignKey: {
|
||||
allowNull: false
|
||||
},
|
||||
onDelete: 'CASCADE',
|
||||
hooks: true
|
||||
})
|
||||
RedundancyVideos: VideoRedundancyModel[]
|
||||
|
||||
static isInfohashExists (infoHash: string) {
|
||||
const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
|
||||
const options = {
|
||||
|
@ -27,13 +27,13 @@ import {
|
||||
Table,
|
||||
UpdatedAt
|
||||
} from 'sequelize-typescript'
|
||||
import { VideoPrivacy, VideoResolution, VideoState } from '../../../shared'
|
||||
import { ActivityUrlObject, VideoPrivacy, VideoResolution, VideoState } from '../../../shared'
|
||||
import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
|
||||
import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos'
|
||||
import { VideoFilter } from '../../../shared/models/videos/video-query.type'
|
||||
import { createTorrentPromise, peertubeTruncate } from '../../helpers/core-utils'
|
||||
import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
|
||||
import { isBooleanValid } from '../../helpers/custom-validators/misc'
|
||||
import { isArray, isBooleanValid } from '../../helpers/custom-validators/misc'
|
||||
import {
|
||||
isVideoCategoryValid,
|
||||
isVideoDescriptionValid,
|
||||
@ -90,6 +90,7 @@ import { VideoCaptionModel } from './video-caption'
|
||||
import { VideoBlacklistModel } from './video-blacklist'
|
||||
import { copy, remove, rename, stat, writeFile } from 'fs-extra'
|
||||
import { VideoViewModel } from './video-views'
|
||||
import { VideoRedundancyModel } from '../redundancy/video-redundancy'
|
||||
|
||||
// FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation
|
||||
const indexes: Sequelize.DefineIndexesOptions[] = [
|
||||
@ -470,7 +471,13 @@ type AvailableForListIDsOptions = {
|
||||
include: [
|
||||
{
|
||||
model: () => VideoFileModel.unscoped(),
|
||||
required: false
|
||||
required: false,
|
||||
include: [
|
||||
{
|
||||
model: () => VideoRedundancyModel.unscoped(),
|
||||
required: false
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
@ -633,6 +640,7 @@ export class VideoModel extends Model<VideoModel> {
|
||||
name: 'videoId',
|
||||
allowNull: false
|
||||
},
|
||||
hooks: true,
|
||||
onDelete: 'cascade'
|
||||
})
|
||||
VideoFiles: VideoFileModel[]
|
||||
@ -1325,9 +1333,7 @@ export class VideoModel extends Model<VideoModel> {
|
||||
[ CONFIG.WEBSERVER.WS + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT + '/tracker/socket' ],
|
||||
[ CONFIG.WEBSERVER.URL + '/tracker/announce' ]
|
||||
],
|
||||
urlList: [
|
||||
CONFIG.WEBSERVER.URL + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile)
|
||||
]
|
||||
urlList: [ CONFIG.WEBSERVER.URL + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile) ]
|
||||
}
|
||||
|
||||
const torrent = await createTorrentPromise(this.getVideoFilePath(videoFile), options)
|
||||
@ -1535,11 +1541,11 @@ export class VideoModel extends Model<VideoModel> {
|
||||
}
|
||||
}
|
||||
|
||||
const url = []
|
||||
const url: ActivityUrlObject[] = []
|
||||
for (const file of this.VideoFiles) {
|
||||
url.push({
|
||||
type: 'Link',
|
||||
mimeType: VIDEO_EXT_MIMETYPE[ file.extname ],
|
||||
mimeType: VIDEO_EXT_MIMETYPE[ file.extname ] as any,
|
||||
href: this.getVideoFileUrl(file, baseUrlHttp),
|
||||
height: file.resolution,
|
||||
size: file.size,
|
||||
@ -1548,14 +1554,14 @@ export class VideoModel extends Model<VideoModel> {
|
||||
|
||||
url.push({
|
||||
type: 'Link',
|
||||
mimeType: 'application/x-bittorrent',
|
||||
mimeType: 'application/x-bittorrent' as 'application/x-bittorrent',
|
||||
href: this.getTorrentUrl(file, baseUrlHttp),
|
||||
height: file.resolution
|
||||
})
|
||||
|
||||
url.push({
|
||||
type: 'Link',
|
||||
mimeType: 'application/x-bittorrent;x-scheme-handler/magnet',
|
||||
mimeType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet',
|
||||
href: this.generateMagnetUri(file, baseUrlHttp, baseUrlWs),
|
||||
height: file.resolution
|
||||
})
|
||||
@ -1796,7 +1802,7 @@ export class VideoModel extends Model<VideoModel> {
|
||||
(now - updatedAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL
|
||||
}
|
||||
|
||||
private getBaseUrls () {
|
||||
getBaseUrls () {
|
||||
let baseUrlHttp
|
||||
let baseUrlWs
|
||||
|
||||
@ -1811,30 +1817,13 @@ export class VideoModel extends Model<VideoModel> {
|
||||
return { baseUrlHttp, baseUrlWs }
|
||||
}
|
||||
|
||||
private getThumbnailUrl (baseUrlHttp: string) {
|
||||
return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName()
|
||||
}
|
||||
|
||||
private getTorrentUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
|
||||
return baseUrlHttp + STATIC_PATHS.TORRENTS + this.getTorrentFileName(videoFile)
|
||||
}
|
||||
|
||||
private getTorrentDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
|
||||
return baseUrlHttp + STATIC_DOWNLOAD_PATHS.TORRENTS + this.getTorrentFileName(videoFile)
|
||||
}
|
||||
|
||||
private getVideoFileUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
|
||||
return baseUrlHttp + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile)
|
||||
}
|
||||
|
||||
private getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
|
||||
return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile)
|
||||
}
|
||||
|
||||
private generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) {
|
||||
generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) {
|
||||
const xs = this.getTorrentUrl(videoFile, baseUrlHttp)
|
||||
const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ]
|
||||
const urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ]
|
||||
let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ]
|
||||
|
||||
const redundancies = videoFile.RedundancyVideos
|
||||
if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl))
|
||||
|
||||
const magnetHash = {
|
||||
xs,
|
||||
@ -1846,4 +1835,24 @@ export class VideoModel extends Model<VideoModel> {
|
||||
|
||||
return magnetUtil.encode(magnetHash)
|
||||
}
|
||||
|
||||
getThumbnailUrl (baseUrlHttp: string) {
|
||||
return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName()
|
||||
}
|
||||
|
||||
getTorrentUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
|
||||
return baseUrlHttp + STATIC_PATHS.TORRENTS + this.getTorrentFileName(videoFile)
|
||||
}
|
||||
|
||||
getTorrentDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
|
||||
return baseUrlHttp + STATIC_DOWNLOAD_PATHS.TORRENTS + this.getTorrentFileName(videoFile)
|
||||
}
|
||||
|
||||
getVideoFileUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
|
||||
return baseUrlHttp + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile)
|
||||
}
|
||||
|
||||
getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
|
||||
return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile)
|
||||
}
|
||||
}
|
||||
|
@ -169,15 +169,6 @@ describe('Test server follows API validators', function () {
|
||||
statusCodeExpected: 404
|
||||
})
|
||||
})
|
||||
|
||||
it('Should succeed with the correct parameters', async function () {
|
||||
await makeDeleteRequest({
|
||||
url: server.url,
|
||||
path: path + '/localhost:9002',
|
||||
token: server.accessToken,
|
||||
statusCodeExpected: 404
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
|
@ -1,15 +1,17 @@
|
||||
// Order of the tests we want to execute
|
||||
import './accounts'
|
||||
import './config'
|
||||
import './follows'
|
||||
import './jobs'
|
||||
import './redundancy'
|
||||
import './search'
|
||||
import './services'
|
||||
import './user-subscriptions'
|
||||
import './users'
|
||||
import './video-abuses'
|
||||
import './video-blacklist'
|
||||
import './video-captions'
|
||||
import './video-channels'
|
||||
import './video-comments'
|
||||
import './videos'
|
||||
import './video-imports'
|
||||
import './search'
|
||||
import './user-subscriptions'
|
||||
import './videos'
|
||||
|
103
server/tests/api/check-params/redundancy.ts
Normal file
103
server/tests/api/check-params/redundancy.ts
Normal file
@ -0,0 +1,103 @@
|
||||
/* tslint:disable:no-unused-expression */
|
||||
|
||||
import 'mocha'
|
||||
|
||||
import {
|
||||
createUser,
|
||||
doubleFollow,
|
||||
flushAndRunMultipleServers,
|
||||
flushTests,
|
||||
killallServers,
|
||||
makePutBodyRequest,
|
||||
ServerInfo,
|
||||
setAccessTokensToServers,
|
||||
userLogin
|
||||
} from '../../utils'
|
||||
|
||||
describe('Test server redundancy API validators', function () {
|
||||
let servers: ServerInfo[]
|
||||
let userAccessToken = null
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
before(async function () {
|
||||
this.timeout(30000)
|
||||
|
||||
await flushTests()
|
||||
servers = await flushAndRunMultipleServers(2)
|
||||
|
||||
await setAccessTokensToServers(servers)
|
||||
await doubleFollow(servers[0], servers[1])
|
||||
|
||||
const user = {
|
||||
username: 'user1',
|
||||
password: 'password'
|
||||
}
|
||||
|
||||
await createUser(servers[0].url, servers[0].accessToken, user.username, user.password)
|
||||
userAccessToken = await userLogin(servers[0], user)
|
||||
})
|
||||
|
||||
describe('When updating redundancy', function () {
|
||||
const path = '/api/v1/server/redundancy'
|
||||
|
||||
it('Should fail with an invalid token', async function () {
|
||||
await makePutBodyRequest({
|
||||
url: servers[0].url,
|
||||
path: path + '/localhost:9002',
|
||||
fields: { redundancyAllowed: true },
|
||||
token: 'fake_token',
|
||||
statusCodeExpected: 401
|
||||
})
|
||||
})
|
||||
|
||||
it('Should fail if the user is not an administrator', async function () {
|
||||
await makePutBodyRequest({
|
||||
url: servers[0].url,
|
||||
path: path + '/localhost:9002',
|
||||
fields: { redundancyAllowed: true },
|
||||
token: userAccessToken,
|
||||
statusCodeExpected: 403
|
||||
})
|
||||
})
|
||||
|
||||
it('Should fail if we do not follow this server', async function () {
|
||||
await makePutBodyRequest({
|
||||
url: servers[0].url,
|
||||
path: path + '/example.com',
|
||||
fields: { redundancyAllowed: true },
|
||||
token: servers[0].accessToken,
|
||||
statusCodeExpected: 404
|
||||
})
|
||||
})
|
||||
|
||||
it('Should fail without de redundancyAllowed param', async function () {
|
||||
await makePutBodyRequest({
|
||||
url: servers[0].url,
|
||||
path: path + '/localhost:9002',
|
||||
fields: { blabla: true },
|
||||
token: servers[0].accessToken,
|
||||
statusCodeExpected: 400
|
||||
})
|
||||
})
|
||||
|
||||
it('Should succeed with the correct parameters', async function () {
|
||||
await makePutBodyRequest({
|
||||
url: servers[0].url,
|
||||
path: path + '/localhost:9002',
|
||||
fields: { redundancyAllowed: true },
|
||||
token: servers[0].accessToken,
|
||||
statusCodeExpected: 204
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
after(async function () {
|
||||
killallServers(servers)
|
||||
|
||||
// Keep the logs if the test failed
|
||||
if (this['ok']) {
|
||||
await flushTests()
|
||||
}
|
||||
})
|
||||
})
|
@ -3,6 +3,7 @@ import './email'
|
||||
import './follows'
|
||||
import './handle-down'
|
||||
import './jobs'
|
||||
import './redundancy'
|
||||
import './reverse-proxy'
|
||||
import './stats'
|
||||
import './tracker'
|
||||
|
140
server/tests/api/server/redundancy.ts
Normal file
140
server/tests/api/server/redundancy.ts
Normal file
@ -0,0 +1,140 @@
|
||||
/* tslint:disable:no-unused-expression */
|
||||
|
||||
import * as chai from 'chai'
|
||||
import 'mocha'
|
||||
import { VideoDetails } from '../../../../shared/models/videos'
|
||||
import {
|
||||
doubleFollow,
|
||||
flushAndRunMultipleServers,
|
||||
flushTests,
|
||||
getFollowingListPaginationAndSort,
|
||||
getVideo,
|
||||
killallServers,
|
||||
ServerInfo,
|
||||
setAccessTokensToServers,
|
||||
uploadVideo,
|
||||
wait,
|
||||
root, viewVideo
|
||||
} from '../../utils'
|
||||
import { waitJobs } from '../../utils/server/jobs'
|
||||
import * as magnetUtil from 'magnet-uri'
|
||||
import { updateRedundancy } from '../../utils/server/redundancy'
|
||||
import { ActorFollow } from '../../../../shared/models/actors'
|
||||
import { readdir } from 'fs-extra'
|
||||
import { join } from 'path'
|
||||
|
||||
const expect = chai.expect
|
||||
|
||||
function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[]) {
|
||||
const parsed = magnetUtil.decode(file.magnetUri)
|
||||
|
||||
for (const ws of baseWebseeds) {
|
||||
const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
|
||||
expect(found, `Webseed ${ws} not found in ${file.magnetUri}`).to.not.be.undefined
|
||||
}
|
||||
}
|
||||
|
||||
describe('Test videos redundancy', function () {
|
||||
let servers: ServerInfo[] = []
|
||||
let video1Server2UUID: string
|
||||
let video2Server2UUID: string
|
||||
|
||||
before(async function () {
|
||||
this.timeout(120000)
|
||||
|
||||
servers = await flushAndRunMultipleServers(3)
|
||||
|
||||
// Get the access tokens
|
||||
await setAccessTokensToServers(servers)
|
||||
|
||||
{
|
||||
const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
|
||||
video1Server2UUID = res.body.video.uuid
|
||||
|
||||
await viewVideo(servers[1].url, video1Server2UUID)
|
||||
}
|
||||
|
||||
{
|
||||
const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
|
||||
video2Server2UUID = res.body.video.uuid
|
||||
}
|
||||
|
||||
await waitJobs(servers)
|
||||
|
||||
// Server 1 and server 2 follow each other
|
||||
await doubleFollow(servers[0], servers[1])
|
||||
// Server 1 and server 3 follow each other
|
||||
await doubleFollow(servers[0], servers[2])
|
||||
// Server 2 and server 3 follow each other
|
||||
await doubleFollow(servers[1], servers[2])
|
||||
|
||||
await waitJobs(servers)
|
||||
})
|
||||
|
||||
it('Should have 1 webseed on the first video', async function () {
|
||||
const webseeds = [
|
||||
'http://localhost:9002/static/webseed/' + video1Server2UUID
|
||||
]
|
||||
|
||||
for (const server of servers) {
|
||||
const res = await getVideo(server.url, video1Server2UUID)
|
||||
|
||||
const video: VideoDetails = res.body
|
||||
video.files.forEach(f => checkMagnetWebseeds(f, webseeds))
|
||||
}
|
||||
})
|
||||
|
||||
it('Should enable redundancy on server 1', async function () {
|
||||
await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true)
|
||||
|
||||
const res = await getFollowingListPaginationAndSort(servers[0].url, 0, 5, '-createdAt')
|
||||
const follows: ActorFollow[] = res.body.data
|
||||
const server2 = follows.find(f => f.following.host === 'localhost:9002')
|
||||
const server3 = follows.find(f => f.following.host === 'localhost:9003')
|
||||
|
||||
expect(server3).to.not.be.undefined
|
||||
expect(server3.following.hostRedundancyAllowed).to.be.false
|
||||
|
||||
expect(server2).to.not.be.undefined
|
||||
expect(server2.following.hostRedundancyAllowed).to.be.true
|
||||
})
|
||||
|
||||
it('Should have 2 webseed on the first video', async function () {
|
||||
this.timeout(40000)
|
||||
|
||||
await waitJobs(servers)
|
||||
await wait(15000)
|
||||
await waitJobs(servers)
|
||||
|
||||
const webseeds = [
|
||||
'http://localhost:9001/static/webseed/' + video1Server2UUID,
|
||||
'http://localhost:9002/static/webseed/' + video1Server2UUID
|
||||
]
|
||||
|
||||
for (const server of servers) {
|
||||
const res = await getVideo(server.url, video1Server2UUID)
|
||||
|
||||
const video: VideoDetails = res.body
|
||||
|
||||
for (const file of video.files) {
|
||||
checkMagnetWebseeds(file, webseeds)
|
||||
}
|
||||
}
|
||||
|
||||
const files = await readdir(join(root(), 'test1', 'videos'))
|
||||
expect(files).to.have.lengthOf(4)
|
||||
|
||||
for (const resolution of [ 240, 360, 480, 720 ]) {
|
||||
expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined
|
||||
}
|
||||
})
|
||||
|
||||
after(async function () {
|
||||
killallServers(servers)
|
||||
|
||||
// Keep the logs if the test failed
|
||||
if (this['ok']) {
|
||||
await flushTests()
|
||||
}
|
||||
})
|
||||
})
|
@ -1,5 +1,4 @@
|
||||
import * as request from 'supertest'
|
||||
import { wait } from '../miscs/miscs'
|
||||
import { ServerInfo } from './servers'
|
||||
import { waitJobs } from './jobs'
|
||||
|
||||
|
17
server/tests/utils/server/redundancy.ts
Normal file
17
server/tests/utils/server/redundancy.ts
Normal file
@ -0,0 +1,17 @@
|
||||
import { makePutBodyRequest } from '../requests/requests'
|
||||
|
||||
async function updateRedundancy (url: string, accessToken: string, host: string, redundancyAllowed: boolean, expectedStatus = 204) {
|
||||
const path = '/api/v1/server/redundancy/' + host
|
||||
|
||||
return makePutBodyRequest({
|
||||
url,
|
||||
path,
|
||||
token: accessToken,
|
||||
fields: { redundancyAllowed },
|
||||
statusCodeExpected: expectedStatus
|
||||
})
|
||||
}
|
||||
|
||||
export {
|
||||
updateRedundancy
|
||||
}
|
@ -1,6 +1,6 @@
|
||||
import { ActivityPubActor } from './activitypub-actor'
|
||||
import { ActivityPubSignature } from './activitypub-signature'
|
||||
import { VideoTorrentObject } from './objects'
|
||||
import { CacheFileObject, VideoTorrentObject } from './objects'
|
||||
import { DislikeObject } from './objects/dislike-object'
|
||||
import { VideoAbuseObject } from './objects/video-abuse-object'
|
||||
import { VideoCommentObject } from './objects/video-comment-object'
|
||||
@ -29,12 +29,12 @@ export interface BaseActivity {
|
||||
|
||||
export interface ActivityCreate extends BaseActivity {
|
||||
type: 'Create'
|
||||
object: VideoTorrentObject | VideoAbuseObject | ViewObject | DislikeObject | VideoCommentObject
|
||||
object: VideoTorrentObject | VideoAbuseObject | ViewObject | DislikeObject | VideoCommentObject | CacheFileObject
|
||||
}
|
||||
|
||||
export interface ActivityUpdate extends BaseActivity {
|
||||
type: 'Update'
|
||||
object: VideoTorrentObject | ActivityPubActor
|
||||
object: VideoTorrentObject | ActivityPubActor | CacheFileObject
|
||||
}
|
||||
|
||||
export interface ActivityDelete extends BaseActivity {
|
||||
|
9
shared/models/activitypub/objects/cache-file-object.ts
Normal file
9
shared/models/activitypub/objects/cache-file-object.ts
Normal file
@ -0,0 +1,9 @@
|
||||
import { ActivityVideoUrlObject } from './common-objects'
|
||||
|
||||
export interface CacheFileObject {
|
||||
id: string
|
||||
type: 'CacheFile',
|
||||
object: string
|
||||
expires: string
|
||||
url: ActivityVideoUrlObject
|
||||
}
|
@ -17,16 +17,31 @@ export interface ActivityIconObject {
|
||||
height: number
|
||||
}
|
||||
|
||||
export interface ActivityUrlObject {
|
||||
export type ActivityVideoUrlObject = {
|
||||
type: 'Link'
|
||||
mimeType: 'video/mp4' | 'video/webm' | 'application/x-bittorrent' | 'application/x-bittorrent;x-scheme-handler/magnet'
|
||||
mimeType: 'video/mp4' | 'video/webm' | 'video/ogg'
|
||||
href: string
|
||||
height: number
|
||||
|
||||
size?: number
|
||||
fps?: number
|
||||
size: number
|
||||
fps: number
|
||||
}
|
||||
|
||||
export type ActivityUrlObject =
|
||||
ActivityVideoUrlObject
|
||||
|
|
||||
{
|
||||
type: 'Link'
|
||||
mimeType: 'application/x-bittorrent' | 'application/x-bittorrent;x-scheme-handler/magnet'
|
||||
href: string
|
||||
height: number
|
||||
}
|
||||
|
|
||||
{
|
||||
type: 'Link'
|
||||
mimeType: 'text/html'
|
||||
href: string
|
||||
}
|
||||
|
||||
export interface ActivityPubAttributedTo {
|
||||
type: 'Group' | 'Person'
|
||||
id: string
|
||||
|
@ -1,3 +1,4 @@
|
||||
export * from './cache-file-object'
|
||||
export * from './common-objects'
|
||||
export * from './video-abuse-object'
|
||||
export * from './video-torrent-object'
|
||||
|
@ -1,10 +1,10 @@
|
||||
import {
|
||||
ActivityIconObject,
|
||||
ActivityIdentifierObject, ActivityPubAttributedTo,
|
||||
ActivityIdentifierObject,
|
||||
ActivityPubAttributedTo,
|
||||
ActivityTagObject,
|
||||
ActivityUrlObject
|
||||
} from './common-objects'
|
||||
import { ActivityPubOrderedCollection } from '../activitypub-ordered-collection'
|
||||
import { VideoState } from '../../videos'
|
||||
|
||||
export interface VideoTorrentObject {
|
||||
|
@ -2,10 +2,10 @@ import { Actor } from './actor.model'
|
||||
|
||||
export type FollowState = 'pending' | 'accepted'
|
||||
|
||||
export interface AccountFollow {
|
||||
export interface ActorFollow {
|
||||
id: number
|
||||
follower: Actor
|
||||
following: Actor
|
||||
follower: Actor & { hostRedundancyAllowed: boolean }
|
||||
following: Actor & { hostRedundancyAllowed: boolean }
|
||||
score: number
|
||||
state: FollowState
|
||||
createdAt: Date
|
||||
|
1
shared/models/avatars/index.ts
Normal file
1
shared/models/avatars/index.ts
Normal file
@ -0,0 +1 @@
|
||||
export * from './avatar.model'
|
@ -1,5 +1,7 @@
|
||||
export * from './actors'
|
||||
export * from './activitypub'
|
||||
export * from './actors'
|
||||
export * from './avatars'
|
||||
export * from './redundancy'
|
||||
export * from './users'
|
||||
export * from './videos'
|
||||
export * from './feeds'
|
||||
|
1
shared/models/redundancy/index.ts
Normal file
1
shared/models/redundancy/index.ts
Normal file
@ -0,0 +1 @@
|
||||
export * from './videos-redundancy.model'
|
6
shared/models/redundancy/videos-redundancy.model.ts
Normal file
6
shared/models/redundancy/videos-redundancy.model.ts
Normal file
@ -0,0 +1,6 @@
|
||||
export type VideoRedundancyStrategy = 'most-views'
|
||||
|
||||
export interface VideosRedundancy {
|
||||
strategy: VideoRedundancyStrategy
|
||||
size: number
|
||||
}
|
@ -3,6 +3,7 @@ export enum UserRight {
|
||||
|
||||
MANAGE_USERS,
|
||||
MANAGE_SERVER_FOLLOW,
|
||||
MANAGE_SERVER_REDUNDANCY,
|
||||
MANAGE_VIDEO_ABUSES,
|
||||
MANAGE_JOBS,
|
||||
MANAGE_CONFIGURATION,
|
||||
|
@ -44,6 +44,10 @@
|
||||
"@types/bluebird" "*"
|
||||
"@types/ioredis" "*"
|
||||
|
||||
"@types/bytes@^3.0.0":
|
||||
version "3.0.0"
|
||||
resolved "https://registry.yarnpkg.com/@types/bytes/-/bytes-3.0.0.tgz#549eeacd0a8fecfaa459334583a4edcee738e6db"
|
||||
|
||||
"@types/caseless@*":
|
||||
version "0.12.1"
|
||||
resolved "https://registry.yarnpkg.com/@types/caseless/-/caseless-0.12.1.tgz#9794c69c8385d0192acc471a540d1f8e0d16218a"
|
||||
@ -993,7 +997,7 @@ bytes@1:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/bytes/-/bytes-1.0.0.tgz#3569ede8ba34315fab99c3e92cb04c7220de1fa8"
|
||||
|
||||
bytes@3.0.0:
|
||||
bytes@3.0.0, bytes@^3.0.0:
|
||||
version "3.0.0"
|
||||
resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.0.0.tgz#d32815404d689699f85a4ea4fa8755dd13a96048"
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user