Put private videos under a specific subdirectory

This commit is contained in:
Chocobozzz 2022-10-12 16:09:02 +02:00 committed by Chocobozzz
parent 38a3ccc7f8
commit 3545e72c68
105 changed files with 2929 additions and 1308 deletions

View File

@ -20,12 +20,12 @@ import {
} from '@app/core'
import { HooksService } from '@app/core/plugins/hooks.service'
import { isXPercentInViewport, scrollToTop } from '@app/helpers'
import { Video, VideoCaptionService, VideoDetails, VideoService } from '@app/shared/shared-main'
import { Video, VideoCaptionService, VideoDetails, VideoFileTokenService, VideoService } from '@app/shared/shared-main'
import { SubscribeButtonComponent } from '@app/shared/shared-user-subscription'
import { LiveVideoService } from '@app/shared/shared-video-live'
import { VideoPlaylist, VideoPlaylistService } from '@app/shared/shared-video-playlist'
import { logger } from '@root-helpers/logger'
import { isP2PEnabled } from '@root-helpers/video'
import { isP2PEnabled, videoRequiresAuth } from '@root-helpers/video'
import { timeToInt } from '@shared/core-utils'
import {
HTMLServerConfig,
@ -78,6 +78,8 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
private nextVideoUUID = ''
private nextVideoTitle = ''
private videoFileToken: string
private currentTime: number
private paramsSub: Subscription
@ -110,6 +112,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
private pluginService: PluginService,
private peertubeSocket: PeerTubeSocket,
private screenService: ScreenService,
private videoFileTokenService: VideoFileTokenService,
private location: PlatformLocation,
@Inject(LOCALE_ID) private localeId: string
) { }
@ -252,12 +255,19 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
'filter:api.video-watch.video.get.result'
)
const videoAndLiveObs: Observable<{ video: VideoDetails, live?: LiveVideo }> = videoObs.pipe(
const videoAndLiveObs: Observable<{ video: VideoDetails, live?: LiveVideo, videoFileToken?: string }> = videoObs.pipe(
switchMap(video => {
if (!video.isLive) return of({ video })
if (!video.isLive) return of({ video, live: undefined })
return this.liveVideoService.getVideoLive(video.uuid)
.pipe(map(live => ({ live, video })))
}),
switchMap(({ video, live }) => {
if (!videoRequiresAuth(video)) return of({ video, live, videoFileToken: undefined })
return this.videoFileTokenService.getVideoFileToken(video.uuid)
.pipe(map(({ token }) => ({ video, live, videoFileToken: token })))
})
)
@ -266,7 +276,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
this.videoCaptionService.listCaptions(videoId),
this.userService.getAnonymousOrLoggedUser()
]).subscribe({
next: ([ { video, live }, captionsResult, loggedInOrAnonymousUser ]) => {
next: ([ { video, live, videoFileToken }, captionsResult, loggedInOrAnonymousUser ]) => {
const queryParams = this.route.snapshot.queryParams
const urlOptions = {
@ -283,7 +293,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
peertubeLink: false
}
this.onVideoFetched({ video, live, videoCaptions: captionsResult.data, loggedInOrAnonymousUser, urlOptions })
this.onVideoFetched({ video, live, videoCaptions: captionsResult.data, videoFileToken, loggedInOrAnonymousUser, urlOptions })
.catch(err => this.handleGlobalError(err))
},
@ -356,16 +366,19 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
video: VideoDetails
live: LiveVideo
videoCaptions: VideoCaption[]
videoFileToken: string
urlOptions: URLOptions
loggedInOrAnonymousUser: User
}) {
const { video, live, videoCaptions, urlOptions, loggedInOrAnonymousUser } = options
const { video, live, videoCaptions, urlOptions, videoFileToken, loggedInOrAnonymousUser } = options
this.subscribeToLiveEventsIfNeeded(this.video, video)
this.video = video
this.videoCaptions = videoCaptions
this.liveVideo = live
this.videoFileToken = videoFileToken
// Re init attributes
this.playerPlaceholderImgSrc = undefined
@ -414,6 +427,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
video: this.video,
videoCaptions: this.videoCaptions,
liveVideo: this.liveVideo,
videoFileToken: this.videoFileToken,
urlOptions,
loggedInOrAnonymousUser,
user: this.user
@ -561,11 +575,15 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
video: VideoDetails
liveVideo: LiveVideo
videoCaptions: VideoCaption[]
videoFileToken: string
urlOptions: CustomizationOptions & { playerMode: PlayerMode }
loggedInOrAnonymousUser: User
user?: AuthUser // Keep for plugins
}) {
const { video, liveVideo, videoCaptions, urlOptions, loggedInOrAnonymousUser } = params
const { video, liveVideo, videoCaptions, videoFileToken, urlOptions, loggedInOrAnonymousUser } = params
const getStartTime = () => {
const byUrl = urlOptions.startTime !== undefined
@ -623,13 +641,6 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
theaterButton: true,
captions: videoCaptions.length !== 0,
videoViewUrl: video.privacy.id !== VideoPrivacy.PRIVATE
? this.videoService.getVideoViewUrl(video.uuid)
: null,
authorizationHeader: this.authService.getRequestHeaderValue(),
metricsUrl: environment.apiUrl + '/api/v1/metrics/playback',
embedUrl: video.embedUrl,
embedTitle: video.name,
instanceName: this.serverConfig.instance.name,
@ -639,7 +650,17 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
language: this.localeId,
serverUrl: environment.apiUrl,
metricsUrl: environment.apiUrl + '/api/v1/metrics/playback',
videoViewUrl: video.privacy.id !== VideoPrivacy.PRIVATE
? this.videoService.getVideoViewUrl(video.uuid)
: null,
authorizationHeader: () => this.authService.getRequestHeaderValue(),
serverUrl: environment.originServerUrl,
videoFileToken: () => videoFileToken,
requiresAuth: videoRequiresAuth(video),
videoCaptions: playerCaptions,

View File

@ -1,7 +1,7 @@
import { Observable, of } from 'rxjs'
import { map } from 'rxjs/operators'
import { User } from '@app/core/users/user.model'
import { UserTokens } from '@root-helpers/users'
import { OAuthUserTokens } from '@root-helpers/users'
import { hasUserRight } from '@shared/core-utils/users'
import {
MyUser as ServerMyUserModel,
@ -13,33 +13,33 @@ import {
} from '@shared/models'
export class AuthUser extends User implements ServerMyUserModel {
tokens: UserTokens
oauthTokens: OAuthUserTokens
specialPlaylists: MyUserSpecialPlaylist[]
canSeeVideosLink = true
constructor (userHash: Partial<ServerMyUserModel>, hashTokens: Partial<UserTokens>) {
constructor (userHash: Partial<ServerMyUserModel>, hashTokens: Partial<OAuthUserTokens>) {
super(userHash)
this.tokens = new UserTokens(hashTokens)
this.oauthTokens = new OAuthUserTokens(hashTokens)
this.specialPlaylists = userHash.specialPlaylists
}
getAccessToken () {
return this.tokens.accessToken
return this.oauthTokens.accessToken
}
getRefreshToken () {
return this.tokens.refreshToken
return this.oauthTokens.refreshToken
}
getTokenType () {
return this.tokens.tokenType
return this.oauthTokens.tokenType
}
refreshTokens (accessToken: string, refreshToken: string) {
this.tokens.accessToken = accessToken
this.tokens.refreshToken = refreshToken
this.oauthTokens.accessToken = accessToken
this.oauthTokens.refreshToken = refreshToken
}
hasRight (right: UserRight) {

View File

@ -5,7 +5,7 @@ import { HttpClient, HttpErrorResponse, HttpHeaders, HttpParams } from '@angular
import { Injectable } from '@angular/core'
import { Router } from '@angular/router'
import { Notifier } from '@app/core/notification/notifier.service'
import { logger, objectToUrlEncoded, peertubeLocalStorage, UserTokens } from '@root-helpers/index'
import { logger, OAuthUserTokens, objectToUrlEncoded, peertubeLocalStorage } from '@root-helpers/index'
import { HttpStatusCode, MyUser as UserServerModel, OAuthClientLocal, User, UserLogin, UserRefreshToken } from '@shared/models'
import { environment } from '../../../environments/environment'
import { RestExtractor } from '../rest/rest-extractor.service'
@ -74,7 +74,7 @@ export class AuthService {
]
}
buildAuthUser (userInfo: Partial<User>, tokens: UserTokens) {
buildAuthUser (userInfo: Partial<User>, tokens: OAuthUserTokens) {
this.user = new AuthUser(userInfo, tokens)
}

View File

@ -4,7 +4,7 @@ import { Injectable } from '@angular/core'
import { AuthService, AuthStatus } from '@app/core/auth'
import { getBoolOrDefault } from '@root-helpers/local-storage-utils'
import { logger } from '@root-helpers/logger'
import { UserLocalStorageKeys, UserTokens } from '@root-helpers/users'
import { UserLocalStorageKeys, OAuthUserTokens } from '@root-helpers/users'
import { UserRole, UserUpdateMe } from '@shared/models'
import { NSFWPolicyType } from '@shared/models/videos'
import { ServerService } from '../server'
@ -24,7 +24,7 @@ export class UserLocalStorageService {
this.setLoggedInUser(user)
this.setUserInfo(user)
this.setTokens(user.tokens)
this.setTokens(user.oauthTokens)
}
})
@ -43,7 +43,7 @@ export class UserLocalStorageService {
next: () => {
const user = this.authService.getUser()
this.setTokens(user.tokens)
this.setTokens(user.oauthTokens)
}
})
}
@ -174,14 +174,14 @@ export class UserLocalStorageService {
// ---------------------------------------------------------------------------
getTokens () {
return UserTokens.getUserTokens(this.localStorageService)
return OAuthUserTokens.getUserTokens(this.localStorageService)
}
setTokens (tokens: UserTokens) {
UserTokens.saveToLocalStorage(this.localStorageService, tokens)
setTokens (tokens: OAuthUserTokens) {
OAuthUserTokens.saveToLocalStorage(this.localStorageService, tokens)
}
flushTokens () {
UserTokens.flushLocalStorage(this.localStorageService)
OAuthUserTokens.flushLocalStorage(this.localStorageService)
}
}

View File

@ -54,8 +54,9 @@ function objectToFormData (obj: any, form?: FormData, namespace?: string) {
}
export {
objectToFormData,
getAbsoluteAPIUrl,
getAPIHost,
getAbsoluteEmbedUrl
getAbsoluteEmbedUrl,
objectToFormData
}

View File

@ -44,7 +44,15 @@ import {
import { PluginPlaceholderComponent, PluginSelectorDirective } from './plugins'
import { ActorRedirectGuard } from './router'
import { UserHistoryService, UserNotificationsComponent, UserNotificationService, UserQuotaComponent } from './users'
import { EmbedComponent, RedundancyService, VideoImportService, VideoOwnershipService, VideoResolver, VideoService } from './video'
import {
EmbedComponent,
RedundancyService,
VideoFileTokenService,
VideoImportService,
VideoOwnershipService,
VideoResolver,
VideoService
} from './video'
import { VideoCaptionService } from './video-caption'
import { VideoChannelService } from './video-channel'
@ -185,6 +193,7 @@ import { VideoChannelService } from './video-channel'
VideoImportService,
VideoOwnershipService,
VideoService,
VideoFileTokenService,
VideoResolver,
VideoCaptionService,

View File

@ -2,6 +2,7 @@ export * from './embed.component'
export * from './redundancy.service'
export * from './video-details.model'
export * from './video-edit.model'
export * from './video-file-token.service'
export * from './video-import.service'
export * from './video-ownership.service'
export * from './video.model'

View File

@ -0,0 +1,33 @@
import { catchError, map, of, tap } from 'rxjs'
import { HttpClient } from '@angular/common/http'
import { Injectable } from '@angular/core'
import { RestExtractor } from '@app/core'
import { VideoToken } from '@shared/models'
import { VideoService } from './video.service'
@Injectable()
export class VideoFileTokenService {
private readonly store = new Map<string, { token: string, expires: Date }>()
constructor (
private authHttp: HttpClient,
private restExtractor: RestExtractor
) {}
getVideoFileToken (videoUUID: string) {
const existing = this.store.get(videoUUID)
if (existing) return of(existing)
return this.createVideoFileToken(videoUUID)
.pipe(tap(result => this.store.set(videoUUID, { token: result.token, expires: new Date(result.expires) })))
}
private createVideoFileToken (videoUUID: string) {
return this.authHttp.post<VideoToken>(`${VideoService.BASE_VIDEO_URL}/${videoUUID}/token`, {})
.pipe(
map(({ files }) => files),
catchError(err => this.restExtractor.handleError(err))
)
}
}

View File

@ -48,10 +48,7 @@
<ng-template ngbNavContent>
<div class="nav-content">
<my-input-text
*ngIf="!isConfidentialVideo()"
[show]="true" [readonly]="true" [withCopy]="true" [withToggle]="false" [value]="getLink()"
></my-input-text>
<my-input-text [show]="true" [readonly]="true" [withCopy]="true" [withToggle]="false" [value]="getLink()"></my-input-text>
</div>
</ng-template>
</ng-container>

View File

@ -2,11 +2,12 @@ import { mapValues, pick } from 'lodash-es'
import { firstValueFrom } from 'rxjs'
import { tap } from 'rxjs/operators'
import { Component, ElementRef, Inject, LOCALE_ID, ViewChild } from '@angular/core'
import { AuthService, HooksService, Notifier } from '@app/core'
import { HooksService } from '@app/core'
import { NgbModal, NgbModalRef } from '@ng-bootstrap/ng-bootstrap'
import { logger } from '@root-helpers/logger'
import { videoRequiresAuth } from '@root-helpers/video'
import { VideoCaption, VideoFile, VideoPrivacy } from '@shared/models'
import { BytesPipe, NumberFormatterPipe, VideoDetails, VideoService } from '../shared-main'
import { BytesPipe, NumberFormatterPipe, VideoDetails, VideoFileTokenService, VideoService } from '../shared-main'
type DownloadType = 'video' | 'subtitles'
type FileMetadata = { [key: string]: { label: string, value: string }}
@ -32,6 +33,8 @@ export class VideoDownloadComponent {
type: DownloadType = 'video'
videoFileToken: string
private activeModal: NgbModalRef
private bytesPipe: BytesPipe
@ -42,10 +45,9 @@ export class VideoDownloadComponent {
constructor (
@Inject(LOCALE_ID) private localeId: string,
private notifier: Notifier,
private modalService: NgbModal,
private videoService: VideoService,
private auth: AuthService,
private videoFileTokenService: VideoFileTokenService,
private hooks: HooksService
) {
this.bytesPipe = new BytesPipe()
@ -71,6 +73,8 @@ export class VideoDownloadComponent {
}
show (video: VideoDetails, videoCaptions?: VideoCaption[]) {
this.videoFileToken = undefined
this.video = video
this.videoCaptions = videoCaptions
@ -84,6 +88,11 @@ export class VideoDownloadComponent {
this.subtitleLanguageId = this.videoCaptions[0].language.id
}
if (videoRequiresAuth(this.video)) {
this.videoFileTokenService.getVideoFileToken(this.video.uuid)
.subscribe(({ token }) => this.videoFileToken = token)
}
this.activeModal.shown.subscribe(() => {
this.hooks.runAction('action:modal.video-download.shown', 'common')
})
@ -155,7 +164,7 @@ export class VideoDownloadComponent {
if (!file) return ''
const suffix = this.isConfidentialVideo()
? '?access_token=' + this.auth.getAccessToken()
? '?videoFileToken=' + this.videoFileToken
: ''
switch (this.downloadType) {

View File

@ -52,6 +52,10 @@ function getRtcConfig () {
}
}
function isSameOrigin (current: string, target: string) {
return new URL(current).origin === new URL(target).origin
}
// ---------------------------------------------------------------------------
export {
@ -60,5 +64,7 @@ export {
videoFileMaxByResolution,
videoFileMinByResolution,
bytes
bytes,
isSameOrigin
}

View File

@ -5,7 +5,7 @@ import { LiveVideoLatencyMode } from '@shared/models'
import { getAverageBandwidthInStore } from '../../peertube-player-local-storage'
import { P2PMediaLoader, P2PMediaLoaderPluginOptions } from '../../types'
import { PeertubePlayerManagerOptions } from '../../types/manager-options'
import { getRtcConfig } from '../common'
import { getRtcConfig, isSameOrigin } from '../common'
import { RedundancyUrlManager } from '../p2p-media-loader/redundancy-url-manager'
import { segmentUrlBuilderFactory } from '../p2p-media-loader/segment-url-builder'
import { segmentValidatorFactory } from '../p2p-media-loader/segment-validator'
@ -84,7 +84,21 @@ export class HLSOptionsBuilder {
simultaneousHttpDownloads: 1,
httpFailedSegmentTimeout: 1000,
segmentValidator: segmentValidatorFactory(this.options.p2pMediaLoader.segmentsSha256Url, this.options.common.isLive),
xhrSetup: (xhr, url) => {
if (!this.options.common.requiresAuth) return
if (!isSameOrigin(this.options.common.serverUrl, url)) return
xhr.setRequestHeader('Authorization', this.options.common.authorizationHeader())
},
segmentValidator: segmentValidatorFactory({
segmentsSha256Url: this.options.p2pMediaLoader.segmentsSha256Url,
isLive: this.options.common.isLive,
authorizationHeader: this.options.common.authorizationHeader,
requiresAuth: this.options.common.requiresAuth,
serverUrl: this.options.common.serverUrl
}),
segmentUrlBuilder: segmentUrlBuilderFactory(redundancyUrlManager),
useP2P: this.options.common.p2pEnabled,

View File

@ -1,4 +1,5 @@
import { PeertubePlayerManagerOptions } from '../../types'
import { addQueryParams } from '../../../../../../shared/core-utils'
import { PeertubePlayerManagerOptions, WebtorrentPluginOptions } from '../../types'
export class WebTorrentOptionsBuilder {
@ -16,13 +17,23 @@ export class WebTorrentOptionsBuilder {
const autoplay = this.autoPlayValue === 'play'
const webtorrent = {
const webtorrent: WebtorrentPluginOptions = {
autoplay,
playerRefusedP2P: commonOptions.p2pEnabled === false,
videoDuration: commonOptions.videoDuration,
playerElement: commonOptions.playerElement,
videoFileToken: commonOptions.videoFileToken,
requiresAuth: commonOptions.requiresAuth,
buildWebSeedUrls: file => {
if (!commonOptions.requiresAuth) return []
return [ addQueryParams(file.fileUrl, { videoFileToken: commonOptions.videoFileToken() }) ]
},
videoFiles: webtorrentOptions.videoFiles.length !== 0
? webtorrentOptions.videoFiles
// The WebTorrent plugin won't be able to play these files, but it will fallback to HTTP mode

View File

@ -2,13 +2,22 @@ import { basename } from 'path'
import { Segment } from '@peertube/p2p-media-loader-core'
import { logger } from '@root-helpers/logger'
import { wait } from '@root-helpers/utils'
import { isSameOrigin } from '../common'
type SegmentsJSON = { [filename: string]: string | { [byterange: string]: string } }
const maxRetries = 3
function segmentValidatorFactory (segmentsSha256Url: string, isLive: boolean) {
let segmentsJSON = fetchSha256Segments(segmentsSha256Url)
function segmentValidatorFactory (options: {
serverUrl: string
segmentsSha256Url: string
isLive: boolean
authorizationHeader: () => string
requiresAuth: boolean
}) {
const { serverUrl, segmentsSha256Url, isLive, authorizationHeader, requiresAuth } = options
let segmentsJSON = fetchSha256Segments({ serverUrl, segmentsSha256Url, authorizationHeader, requiresAuth })
const regex = /bytes=(\d+)-(\d+)/
return async function segmentValidator (segment: Segment, _method: string, _peerId: string, retry = 1) {
@ -28,7 +37,7 @@ function segmentValidatorFactory (segmentsSha256Url: string, isLive: boolean) {
await wait(1000)
segmentsJSON = fetchSha256Segments(segmentsSha256Url)
segmentsJSON = fetchSha256Segments({ serverUrl, segmentsSha256Url, authorizationHeader, requiresAuth })
await segmentValidator(segment, _method, _peerId, retry + 1)
return
@ -68,8 +77,19 @@ export {
// ---------------------------------------------------------------------------
function fetchSha256Segments (url: string) {
return fetch(url)
function fetchSha256Segments (options: {
serverUrl: string
segmentsSha256Url: string
authorizationHeader: () => string
requiresAuth: boolean
}) {
const { serverUrl, segmentsSha256Url, requiresAuth, authorizationHeader } = options
const headers = requiresAuth && isSameOrigin(serverUrl, segmentsSha256Url)
? { Authorization: authorizationHeader() }
: {}
return fetch(segmentsSha256Url, { headers })
.then(res => res.json() as Promise<SegmentsJSON>)
.catch(err => {
logger.error('Cannot get sha256 segments', err)

View File

@ -22,7 +22,7 @@ const Plugin = videojs.getPlugin('plugin')
class PeerTubePlugin extends Plugin {
private readonly videoViewUrl: string
private readonly authorizationHeader: string
private readonly authorizationHeader: () => string
private readonly videoUUID: string
private readonly startTime: number
@ -228,7 +228,7 @@ class PeerTubePlugin extends Plugin {
'Content-type': 'application/json; charset=UTF-8'
})
if (this.authorizationHeader) headers.set('Authorization', this.authorizationHeader)
if (this.authorizationHeader) headers.set('Authorization', this.authorizationHeader())
return fetch(this.videoViewUrl, { method: 'POST', body: JSON.stringify(body), headers })
}

View File

@ -2,7 +2,7 @@ import videojs from 'video.js'
import * as WebTorrent from 'webtorrent'
import { logger } from '@root-helpers/logger'
import { isIOS } from '@root-helpers/web-browser'
import { timeToInt } from '@shared/core-utils'
import { addQueryParams, timeToInt } from '@shared/core-utils'
import { VideoFile } from '@shared/models'
import { getAverageBandwidthInStore, getStoredMute, getStoredVolume, saveAverageBandwidth } from '../../peertube-player-local-storage'
import { PeerTubeResolution, PlayerNetworkInfo, WebtorrentPluginOptions } from '../../types'
@ -38,6 +38,8 @@ class WebTorrentPlugin extends Plugin {
BANDWIDTH_AVERAGE_NUMBER_OF_VALUES: 5 // Last 5 seconds to build average bandwidth
}
private readonly buildWebSeedUrls: (file: VideoFile) => string[]
private readonly webtorrent = new WebTorrent({
tracker: {
rtcConfig: getRtcConfig()
@ -57,6 +59,9 @@ class WebTorrentPlugin extends Plugin {
private isAutoResolutionObservation = false
private playerRefusedP2P = false
private requiresAuth: boolean
private videoFileToken: () => string
private torrentInfoInterval: any
private autoQualityInterval: any
private addTorrentDelay: any
@ -81,6 +86,11 @@ class WebTorrentPlugin extends Plugin {
this.savePlayerSrcFunction = this.player.src
this.playerElement = options.playerElement
this.requiresAuth = options.requiresAuth
this.videoFileToken = options.videoFileToken
this.buildWebSeedUrls = options.buildWebSeedUrls
this.player.ready(() => {
const playerOptions = this.player.options_
@ -268,7 +278,8 @@ class WebTorrentPlugin extends Plugin {
return new CacheChunkStore(new PeertubeChunkStore(chunkLength, storeOpts), {
max: 100
})
}
},
urlList: this.buildWebSeedUrls(this.currentVideoFile)
}
this.torrent = this.webtorrent.add(magnetOrTorrentUrl, torrentOptions, torrent => {
@ -533,7 +544,12 @@ class WebTorrentPlugin extends Plugin {
// Enable error display now this is our last fallback
this.player.one('error', () => this.player.peertube().displayFatalError())
const httpUrl = this.currentVideoFile.fileUrl
let httpUrl = this.currentVideoFile.fileUrl
if (this.requiresAuth && this.videoFileToken) {
httpUrl = addQueryParams(httpUrl, { videoFileToken: this.videoFileToken() })
}
this.player.src = this.savePlayerSrcFunction
this.player.src(httpUrl)

View File

@ -57,7 +57,7 @@ export interface CommonOptions extends CustomizationOptions {
captions: boolean
videoViewUrl: string
authorizationHeader?: string
authorizationHeader?: () => string
metricsUrl: string
@ -77,6 +77,8 @@ export interface CommonOptions extends CustomizationOptions {
videoShortUUID: string
serverUrl: string
requiresAuth: boolean
videoFileToken: () => string
errorNotifier: (message: string) => void
}

View File

@ -95,7 +95,7 @@ type PeerTubePluginOptions = {
videoDuration: number
videoViewUrl: string
authorizationHeader?: string
authorizationHeader?: () => string
subtitle?: string
@ -151,6 +151,11 @@ type WebtorrentPluginOptions = {
startTime: number | string
playerRefusedP2P: boolean
requiresAuth: boolean
videoFileToken: () => string
buildWebSeedUrls: (file: VideoFile) => string[]
}
type P2PMediaLoaderPluginOptions = {

View File

@ -1,6 +1,6 @@
import { ClientLogCreate } from '@shared/models/server'
import { peertubeLocalStorage } from './peertube-web-storage'
import { UserTokens } from './users'
import { OAuthUserTokens } from './users'
export type LoggerHook = (message: LoggerMessage, meta?: LoggerMeta) => void
export type LoggerLevel = 'info' | 'warn' | 'error'
@ -56,7 +56,7 @@ class Logger {
})
try {
const tokens = UserTokens.getUserTokens(peertubeLocalStorage)
const tokens = OAuthUserTokens.getUserTokens(peertubeLocalStorage)
if (tokens) headers.set('Authorization', `${tokens.tokenType} ${tokens.accessToken}`)
} catch (err) {

View File

@ -1,2 +1,2 @@
export * from './user-local-storage-keys'
export * from './user-tokens'
export * from './oauth-user-tokens'

View File

@ -1,11 +1,11 @@
import { UserTokenLocalStorageKeys } from './user-local-storage-keys'
export class UserTokens {
export class OAuthUserTokens {
accessToken: string
refreshToken: string
tokenType: string
constructor (hash?: Partial<UserTokens>) {
constructor (hash?: Partial<OAuthUserTokens>) {
if (hash) {
this.accessToken = hash.accessToken
this.refreshToken = hash.refreshToken
@ -25,14 +25,14 @@ export class UserTokens {
if (!accessTokenLocalStorage || !refreshTokenLocalStorage || !tokenTypeLocalStorage) return null
return new UserTokens({
return new OAuthUserTokens({
accessToken: accessTokenLocalStorage,
refreshToken: refreshTokenLocalStorage,
tokenType: tokenTypeLocalStorage
})
}
static saveToLocalStorage (localStorage: Pick<Storage, 'setItem'>, tokens: UserTokens) {
static saveToLocalStorage (localStorage: Pick<Storage, 'setItem'>, tokens: OAuthUserTokens) {
localStorage.setItem(UserTokenLocalStorageKeys.ACCESS_TOKEN, tokens.accessToken)
localStorage.setItem(UserTokenLocalStorageKeys.REFRESH_TOKEN, tokens.refreshToken)
localStorage.setItem(UserTokenLocalStorageKeys.TOKEN_TYPE, tokens.tokenType)

View File

@ -1,4 +1,4 @@
import { HTMLServerConfig, Video } from '@shared/models'
import { HTMLServerConfig, Video, VideoPrivacy } from '@shared/models'
function buildVideoOrPlaylistEmbed (options: {
embedUrl: string
@ -26,9 +26,14 @@ function isP2PEnabled (video: Video, config: HTMLServerConfig, userP2PEnabled: b
return userP2PEnabled
}
function videoRequiresAuth (video: Video) {
return new Set([ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]).has(video.privacy.id)
}
export {
buildVideoOrPlaylistEmbed,
isP2PEnabled
isP2PEnabled,
videoRequiresAuth
}
// ---------------------------------------------------------------------------

View File

@ -6,7 +6,7 @@ import { peertubeTranslate } from '../../../../shared/core-utils/i18n'
import { HTMLServerConfig, LiveVideo, ResultList, VideoDetails, VideoPlaylist, VideoPlaylistElement } from '../../../../shared/models'
import { PeertubePlayerManager } from '../../assets/player'
import { TranslationsManager } from '../../assets/player/translations-manager'
import { getParamString, logger } from '../../root-helpers'
import { getParamString, logger, videoRequiresAuth } from '../../root-helpers'
import { PeerTubeEmbedApi } from './embed-api'
import { AuthHTTP, LiveManager, PeerTubePlugin, PlayerManagerOptions, PlaylistFetcher, PlaylistTracker, VideoFetcher } from './shared'
import { PlayerHTML } from './shared/player-html'
@ -167,22 +167,25 @@ export class PeerTubeEmbed {
private async buildVideoPlayer (videoResponse: Response, captionsPromise: Promise<Response>) {
const alreadyHadPlayer = this.resetPlayerElement()
const videoInfoPromise: Promise<{ video: VideoDetails, live?: LiveVideo }> = videoResponse.json()
.then((videoInfo: VideoDetails) => {
const videoInfoPromise = videoResponse.json()
.then(async (videoInfo: VideoDetails) => {
this.playerManagerOptions.loadParams(this.config, videoInfo)
if (!alreadyHadPlayer && !this.playerManagerOptions.hasAutoplay()) {
this.playerHTML.buildPlaceholder(videoInfo)
}
const live = videoInfo.isLive
? await this.videoFetcher.loadLive(videoInfo)
: undefined
if (!videoInfo.isLive) {
return { video: videoInfo }
}
const videoFileToken = videoRequiresAuth(videoInfo)
? await this.videoFetcher.loadVideoToken(videoInfo)
: undefined
return this.videoFetcher.loadVideoWithLive(videoInfo)
return { live, video: videoInfo, videoFileToken }
})
const [ { video, live }, translations, captionsResponse, PeertubePlayerManagerModule ] = await Promise.all([
const [ { video, live, videoFileToken }, translations, captionsResponse, PeertubePlayerManagerModule ] = await Promise.all([
videoInfoPromise,
this.translationsPromise,
captionsPromise,
@ -200,6 +203,9 @@ export class PeerTubeEmbed {
translations,
serverConfig: this.config,
authorizationHeader: () => this.http.getHeaderTokenValue(),
videoFileToken: () => videoFileToken,
onVideoUpdate: (uuid: string) => this.loadVideoAndBuildPlayer(uuid),
playlistTracker: this.playlistTracker,

View File

@ -1,5 +1,5 @@
import { HttpStatusCode, OAuth2ErrorCode, UserRefreshToken } from '../../../../../shared/models'
import { objectToUrlEncoded, UserTokens } from '../../../root-helpers'
import { OAuthUserTokens, objectToUrlEncoded } from '../../../root-helpers'
import { peertubeLocalStorage } from '../../../root-helpers/peertube-web-storage'
export class AuthHTTP {
@ -8,30 +8,30 @@ export class AuthHTTP {
CLIENT_SECRET: 'client_secret'
}
private userTokens: UserTokens
private userOAuthTokens: OAuthUserTokens
private headers = new Headers()
constructor () {
this.userTokens = UserTokens.getUserTokens(peertubeLocalStorage)
this.userOAuthTokens = OAuthUserTokens.getUserTokens(peertubeLocalStorage)
if (this.userTokens) this.setHeadersFromTokens()
if (this.userOAuthTokens) this.setHeadersFromTokens()
}
fetch (url: string, { optionalAuth }: { optionalAuth: boolean }) {
fetch (url: string, { optionalAuth, method }: { optionalAuth: boolean, method?: string }) {
const refreshFetchOptions = optionalAuth
? { headers: this.headers }
: {}
return this.refreshFetch(url.toString(), refreshFetchOptions)
return this.refreshFetch(url.toString(), { ...refreshFetchOptions, method })
}
getHeaderTokenValue () {
return `${this.userTokens.tokenType} ${this.userTokens.accessToken}`
return `${this.userOAuthTokens.tokenType} ${this.userOAuthTokens.accessToken}`
}
isLoggedIn () {
return !!this.userTokens
return !!this.userOAuthTokens
}
private refreshFetch (url: string, options?: RequestInit) {
@ -47,7 +47,7 @@ export class AuthHTTP {
headers.set('Content-Type', 'application/x-www-form-urlencoded')
const data = {
refresh_token: this.userTokens.refreshToken,
refresh_token: this.userOAuthTokens.refreshToken,
client_id: clientId,
client_secret: clientSecret,
response_type: 'code',
@ -64,15 +64,15 @@ export class AuthHTTP {
return res.json()
}).then((obj: UserRefreshToken & { code?: OAuth2ErrorCode }) => {
if (!obj || obj.code === OAuth2ErrorCode.INVALID_GRANT) {
UserTokens.flushLocalStorage(peertubeLocalStorage)
OAuthUserTokens.flushLocalStorage(peertubeLocalStorage)
this.removeTokensFromHeaders()
return resolve()
}
this.userTokens.accessToken = obj.access_token
this.userTokens.refreshToken = obj.refresh_token
UserTokens.saveToLocalStorage(peertubeLocalStorage, this.userTokens)
this.userOAuthTokens.accessToken = obj.access_token
this.userOAuthTokens.refreshToken = obj.refresh_token
OAuthUserTokens.saveToLocalStorage(peertubeLocalStorage, this.userOAuthTokens)
this.setHeadersFromTokens()
@ -84,7 +84,7 @@ export class AuthHTTP {
return refreshingTokenPromise
.catch(() => {
UserTokens.flushLocalStorage(peertubeLocalStorage)
OAuthUserTokens.flushLocalStorage(peertubeLocalStorage)
this.removeTokensFromHeaders()
}).then(() => fetch(url, {

View File

@ -17,7 +17,8 @@ import {
isP2PEnabled,
logger,
peertubeLocalStorage,
UserLocalStorageKeys
UserLocalStorageKeys,
videoRequiresAuth
} from '../../../root-helpers'
import { PeerTubePlugin } from './peertube-plugin'
import { PlayerHTML } from './player-html'
@ -154,6 +155,9 @@ export class PlayerManagerOptions {
captionsResponse: Response
live?: LiveVideo
authorizationHeader: () => string
videoFileToken: () => string
serverConfig: HTMLServerConfig
alreadyHadPlayer: boolean
@ -169,9 +173,11 @@ export class PlayerManagerOptions {
video,
captionsResponse,
alreadyHadPlayer,
videoFileToken,
translations,
playlistTracker,
live,
authorizationHeader,
serverConfig
} = options
@ -227,6 +233,10 @@ export class PlayerManagerOptions {
embedUrl: window.location.origin + video.embedPath,
embedTitle: video.name,
requiresAuth: videoRequiresAuth(video),
authorizationHeader,
videoFileToken,
errorNotifier: () => {
// Empty, we don't have a notifier in the embed
},

View File

@ -1,4 +1,4 @@
import { HttpStatusCode, LiveVideo, VideoDetails } from '../../../../../shared/models'
import { HttpStatusCode, LiveVideo, VideoDetails, VideoToken } from '../../../../../shared/models'
import { logger } from '../../../root-helpers'
import { AuthHTTP } from './auth-http'
@ -36,10 +36,15 @@ export class VideoFetcher {
return { captionsPromise, videoResponse }
}
loadVideoWithLive (video: VideoDetails) {
loadLive (video: VideoDetails) {
return this.http.fetch(this.getLiveUrl(video.uuid), { optionalAuth: true })
.then(res => res.json())
.then((live: LiveVideo) => ({ video, live }))
.then(res => res.json() as Promise<LiveVideo>)
}
loadVideoToken (video: VideoDetails) {
return this.http.fetch(this.getVideoTokenUrl(video.uuid), { optionalAuth: true, method: 'POST' })
.then(res => res.json() as Promise<VideoToken>)
.then(token => token.files.token)
}
getVideoViewsUrl (videoUUID: string) {
@ -61,4 +66,8 @@ export class VideoFetcher {
private getLiveUrl (videoId: string) {
return window.location.origin + '/api/v1/videos/live/' + videoId
}
private getVideoTokenUrl (id: string) {
return this.getVideoUrl(id) + '/token'
}
}

View File

@ -103,6 +103,7 @@
"@peertube/http-signature": "^1.7.0",
"@uploadx/core": "^6.0.0",
"async-lru": "^1.1.1",
"async-mutex": "^0.4.0",
"bcrypt": "5.0.1",
"bencode": "^2.0.2",
"bittorrent-tracker": "^9.0.0",
@ -177,7 +178,6 @@
},
"devDependencies": {
"@peertube/maildev": "^1.2.0",
"@types/async-lock": "^1.1.0",
"@types/bcrypt": "^5.0.0",
"@types/bencode": "^2.0.0",
"@types/bluebird": "^3.5.33",

View File

@ -1,74 +0,0 @@
import { pathExists, stat, writeFile } from 'fs-extra'
import parseTorrent from 'parse-torrent'
import { join } from 'path'
import * as Sequelize from 'sequelize'
import { logger } from '@server/helpers/logger'
import { createTorrentPromise } from '@server/helpers/webtorrent'
import { CONFIG } from '@server/initializers/config'
import { HLS_STREAMING_PLAYLIST_DIRECTORY, STATIC_PATHS, WEBSERVER } from '@server/initializers/constants'
import { initDatabaseModels, sequelizeTypescript } from '../../server/initializers/database'
run()
.then(() => process.exit(0))
.catch(err => {
console.error(err)
process.exit(-1)
})
async function run () {
logger.info('Creating torrents and updating database for HSL files.')
await initDatabaseModels(true)
const query = 'select "videoFile".id as id, "videoFile".resolution as resolution, "video".uuid as uuid from "videoFile" ' +
'inner join "videoStreamingPlaylist" ON "videoStreamingPlaylist".id = "videoFile"."videoStreamingPlaylistId" ' +
'inner join video ON video.id = "videoStreamingPlaylist"."videoId" ' +
'WHERE video.remote IS FALSE'
const options = {
type: Sequelize.QueryTypes.SELECT
}
const res = await sequelizeTypescript.query(query, options)
for (const row of res) {
const videoFilename = `${row['uuid']}-${row['resolution']}-fragmented.mp4`
const videoFilePath = join(HLS_STREAMING_PLAYLIST_DIRECTORY, row['uuid'], videoFilename)
logger.info('Processing %s.', videoFilePath)
if (!await pathExists(videoFilePath)) {
console.warn('Cannot generate torrent of %s: file does not exist.', videoFilePath)
continue
}
const createTorrentOptions = {
// Keep the extname, it's used by the client to stream the file inside a web browser
name: `video ${row['uuid']}`,
createdBy: 'PeerTube',
announceList: [
[ WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' ],
[ WEBSERVER.URL + '/tracker/announce' ]
],
urlList: [ WEBSERVER.URL + join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, row['uuid'], videoFilename) ]
}
const torrent = await createTorrentPromise(videoFilePath, createTorrentOptions)
const torrentName = `${row['uuid']}-${row['resolution']}-hls.torrent`
const filePath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentName)
await writeFile(filePath, torrent)
const parsedTorrent = parseTorrent(torrent)
const infoHash = parsedTorrent.infoHash
const stats = await stat(videoFilePath)
const size = stats.size
const queryUpdate = 'UPDATE "videoFile" SET "infoHash" = ?, "size" = ? WHERE id = ?'
const options = {
type: Sequelize.QueryTypes.UPDATE,
replacements: [ infoHash, size, row['id'] ]
}
await sequelizeTypescript.query(queryUpdate, options)
}
}

View File

@ -2,7 +2,7 @@ import { map } from 'bluebird'
import { readdir, remove, stat } from 'fs-extra'
import { basename, join } from 'path'
import { get, start } from 'prompt'
import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants'
import { DIRECTORIES } from '@server/initializers/constants'
import { VideoFileModel } from '@server/models/video/video-file'
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
import { uniqify } from '@shared/core-utils'
@ -37,9 +37,11 @@ async function run () {
console.log('Detecting files to remove, it could take a while...')
toDelete = toDelete.concat(
await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesWebTorrentFileExist()),
await pruneDirectory(DIRECTORIES.VIDEOS.PUBLIC, doesWebTorrentFileExist()),
await pruneDirectory(DIRECTORIES.VIDEOS.PRIVATE, doesWebTorrentFileExist()),
await pruneDirectory(HLS_STREAMING_PLAYLIST_DIRECTORY, doesHLSPlaylistExist()),
await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, doesHLSPlaylistExist()),
await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, doesHLSPlaylistExist()),
await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesTorrentFileExist()),
@ -75,7 +77,7 @@ async function run () {
}
}
type ExistFun = (file: string) => Promise<boolean>
type ExistFun = (file: string) => Promise<boolean> | boolean
async function pruneDirectory (directory: string, existFun: ExistFun) {
const files = await readdir(directory)
@ -92,11 +94,21 @@ async function pruneDirectory (directory: string, existFun: ExistFun) {
}
function doesWebTorrentFileExist () {
return (filePath: string) => VideoFileModel.doesOwnedWebTorrentVideoFileExist(basename(filePath))
return (filePath: string) => {
// Don't delete private directory
if (filePath === DIRECTORIES.VIDEOS.PRIVATE) return true
return VideoFileModel.doesOwnedWebTorrentVideoFileExist(basename(filePath))
}
}
function doesHLSPlaylistExist () {
return (hlsPath: string) => VideoStreamingPlaylistModel.doesOwnedHLSPlaylistExist(basename(hlsPath))
return (hlsPath: string) => {
// Don't delete private directory
if (hlsPath === DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE) return true
return VideoStreamingPlaylistModel.doesOwnedHLSPlaylistExist(basename(hlsPath))
}
}
function doesTorrentFileExist () {
@ -127,8 +139,8 @@ async function doesRedundancyExist (filePath: string) {
const isPlaylist = (await stat(filePath)).isDirectory()
if (isPlaylist) {
// Don't delete HLS directory
if (filePath === HLS_REDUNDANCY_DIRECTORY) return true
// Don't delete HLS redundancy directory
if (filePath === DIRECTORIES.HLS_REDUNDANCY) return true
const uuid = getUUIDFromFilename(filePath)
const video = await VideoModel.loadWithFiles(uuid)

View File

@ -8,6 +8,7 @@ import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
import { UserRight } from '../../../../shared/models/users'
import { authenticate, ensureUserHasRight } from '../../../middlewares'
import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler'
import { UpdateVideosScheduler } from '@server/lib/schedulers/update-videos-scheduler'
const debugRouter = express.Router()
@ -45,6 +46,7 @@ async function runCommand (req: express.Request, res: express.Response) {
'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(),
'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(),
'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(),
'process-update-videos-scheduler': () => UpdateVideosScheduler.Instance.execute(),
'process-video-channel-sync-latest': () => VideoChannelSyncLatestScheduler.Instance.execute()
}

View File

@ -41,6 +41,7 @@ import { ownershipVideoRouter } from './ownership'
import { rateVideoRouter } from './rate'
import { statsRouter } from './stats'
import { studioRouter } from './studio'
import { tokenRouter } from './token'
import { transcodingRouter } from './transcoding'
import { updateRouter } from './update'
import { uploadRouter } from './upload'
@ -63,6 +64,7 @@ videosRouter.use('/', uploadRouter)
videosRouter.use('/', updateRouter)
videosRouter.use('/', filesRouter)
videosRouter.use('/', transcodingRouter)
videosRouter.use('/', tokenRouter)
videosRouter.get('/categories',
openapiOperationDoc({ operationId: 'getCategories' }),

View File

@ -0,0 +1,33 @@
import express from 'express'
import { VideoTokensManager } from '@server/lib/video-tokens-manager'
import { VideoToken } from '@shared/models'
import { asyncMiddleware, authenticate, videosCustomGetValidator } from '../../../middlewares'
const tokenRouter = express.Router()
tokenRouter.post('/:id/token',
authenticate,
asyncMiddleware(videosCustomGetValidator('only-video')),
generateToken
)
// ---------------------------------------------------------------------------
export {
tokenRouter
}
// ---------------------------------------------------------------------------
function generateToken (req: express.Request, res: express.Response) {
const video = res.locals.onlyVideo
const { token, expires } = VideoTokensManager.Instance.create(video.uuid)
return res.json({
files: {
token,
expires
}
} as VideoToken)
}

View File

@ -1,12 +1,12 @@
import express from 'express'
import { Transaction } from 'sequelize/types'
import { changeVideoChannelShare } from '@server/lib/activitypub/share'
import { CreateJobArgument, JobQueue } from '@server/lib/job-queue'
import { buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video'
import { addVideoJobsAfterUpdate, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video'
import { setVideoPrivacy } from '@server/lib/video-privacy'
import { openapiOperationDoc } from '@server/middlewares/doc'
import { FilteredModelAttributes } from '@server/types'
import { MVideoFullLight } from '@server/types/models'
import { HttpStatusCode, ManageVideoTorrentPayload, VideoUpdate } from '@shared/models'
import { HttpStatusCode, VideoUpdate } from '@shared/models'
import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
import { resetSequelizeInstance } from '../../../helpers/database-utils'
import { createReqFiles } from '../../../helpers/express-utils'
@ -18,6 +18,7 @@ import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videosUpdateValidator } from '../../../middlewares'
import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update'
import { VideoModel } from '../../../models/video/video'
import { VideoPathManager } from '@server/lib/video-path-manager'
const lTags = loggerTagsFactory('api', 'video')
const auditLogger = auditLoggerFactory('videos')
@ -47,8 +48,8 @@ async function updateVideo (req: express.Request, res: express.Response) {
const oldVideoAuditView = new VideoAuditView(videoFromReq.toFormattedDetailsJSON())
const videoInfoToUpdate: VideoUpdate = req.body
const wasConfidentialVideo = videoFromReq.isConfidential()
const hadPrivacyForFederation = videoFromReq.hasPrivacyForFederation()
const oldPrivacy = videoFromReq.privacy
const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({
video: videoFromReq,
@ -57,12 +58,13 @@ async function updateVideo (req: express.Request, res: express.Response) {
automaticallyGenerated: false
})
const videoFileLockReleaser = await VideoPathManager.Instance.lockFiles(videoFromReq.uuid)
try {
const { videoInstanceUpdated, isNewVideo } = await sequelizeTypescript.transaction(async t => {
// Refresh video since thumbnails to prevent concurrent updates
const video = await VideoModel.loadFull(videoFromReq.id, t)
const sequelizeOptions = { transaction: t }
const oldVideoChannel = video.VideoChannel
const keysToUpdate: (keyof VideoUpdate & FilteredModelAttributes<VideoModel>)[] = [
@ -97,7 +99,7 @@ async function updateVideo (req: express.Request, res: express.Response) {
await video.setAsRefreshed(t)
}
const videoInstanceUpdated = await video.save(sequelizeOptions) as MVideoFullLight
const videoInstanceUpdated = await video.save({ transaction: t }) as MVideoFullLight
// Thumbnail & preview updates?
if (thumbnailModel) await videoInstanceUpdated.addAndSaveThumbnail(thumbnailModel, t)
@ -113,7 +115,9 @@ async function updateVideo (req: express.Request, res: express.Response) {
await videoInstanceUpdated.$set('VideoChannel', res.locals.videoChannel, { transaction: t })
videoInstanceUpdated.VideoChannel = res.locals.videoChannel
if (hadPrivacyForFederation === true) await changeVideoChannelShare(videoInstanceUpdated, oldVideoChannel, t)
if (hadPrivacyForFederation === true) {
await changeVideoChannelShare(videoInstanceUpdated, oldVideoChannel, t)
}
}
// Schedule an update in the future?
@ -139,7 +143,12 @@ async function updateVideo (req: express.Request, res: express.Response) {
Hooks.runAction('action:api.video.updated', { video: videoInstanceUpdated, body: req.body, req, res })
await addVideoJobsAfterUpdate({ video: videoInstanceUpdated, videoInfoToUpdate, wasConfidentialVideo, isNewVideo })
await addVideoJobsAfterUpdate({
video: videoInstanceUpdated,
nameChanged: !!videoInfoToUpdate.name,
oldPrivacy,
isNewVideo
})
} catch (err) {
// Force fields we want to update
// If the transaction is retried, sequelize will think the object has not changed
@ -147,6 +156,8 @@ async function updateVideo (req: express.Request, res: express.Response) {
resetSequelizeInstance(videoFromReq, videoFieldsSave)
throw err
} finally {
videoFileLockReleaser()
}
return res.type('json')
@ -164,7 +175,7 @@ async function updateVideoPrivacy (options: {
const isNewVideo = videoInstance.isNewVideo(videoInfoToUpdate.privacy)
const newPrivacy = parseInt(videoInfoToUpdate.privacy.toString(), 10)
videoInstance.setPrivacy(newPrivacy)
setVideoPrivacy(videoInstance, newPrivacy)
// Unfederate the video if the new privacy is not compatible with federation
if (hadPrivacyForFederation && !videoInstance.hasPrivacyForFederation()) {
@ -185,50 +196,3 @@ function updateSchedule (videoInstance: MVideoFullLight, videoInfoToUpdate: Vide
return ScheduleVideoUpdateModel.deleteByVideoId(videoInstance.id, transaction)
}
}
async function addVideoJobsAfterUpdate (options: {
video: MVideoFullLight
videoInfoToUpdate: VideoUpdate
wasConfidentialVideo: boolean
isNewVideo: boolean
}) {
const { video, videoInfoToUpdate, wasConfidentialVideo, isNewVideo } = options
const jobs: CreateJobArgument[] = []
if (!video.isLive && videoInfoToUpdate.name) {
for (const file of (video.VideoFiles || [])) {
const payload: ManageVideoTorrentPayload = { action: 'update-metadata', videoId: video.id, videoFileId: file.id }
jobs.push({ type: 'manage-video-torrent', payload })
}
const hls = video.getHLSPlaylist()
for (const file of (hls?.VideoFiles || [])) {
const payload: ManageVideoTorrentPayload = { action: 'update-metadata', streamingPlaylistId: hls.id, videoFileId: file.id }
jobs.push({ type: 'manage-video-torrent', payload })
}
}
jobs.push({
type: 'federate-video',
payload: {
videoUUID: video.uuid,
isNewVideo
}
})
if (wasConfidentialVideo) {
jobs.push({
type: 'notify',
payload: {
action: 'new-video',
videoUUID: video.uuid
}
})
}
return JobQueue.Instance.createSequentialJobFlow(...jobs)
}

View File

@ -7,7 +7,7 @@ import { VideoPathManager } from '@server/lib/video-path-manager'
import { MStreamingPlaylist, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@shared/models'
import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants'
import { asyncMiddleware, videosDownloadValidator } from '../middlewares'
import { asyncMiddleware, optionalAuthenticate, videosDownloadValidator } from '../middlewares'
const downloadRouter = express.Router()
@ -20,12 +20,14 @@ downloadRouter.use(
downloadRouter.use(
STATIC_DOWNLOAD_PATHS.VIDEOS + ':id-:resolution([0-9]+).:extension',
optionalAuthenticate,
asyncMiddleware(videosDownloadValidator),
asyncMiddleware(downloadVideoFile)
)
downloadRouter.use(
STATIC_DOWNLOAD_PATHS.HLS_VIDEOS + ':id-:resolution([0-9]+)-fragmented.:extension',
optionalAuthenticate,
asyncMiddleware(videosDownloadValidator),
asyncMiddleware(downloadHLSVideoFile)
)

View File

@ -1,20 +1,34 @@
import cors from 'cors'
import express from 'express'
import { handleStaticError } from '@server/middlewares'
import {
asyncMiddleware,
ensureCanAccessPrivateVideoHLSFiles,
ensureCanAccessVideoPrivateWebTorrentFiles,
handleStaticError,
optionalAuthenticate
} from '@server/middlewares'
import { CONFIG } from '../initializers/config'
import { HLS_STREAMING_PLAYLIST_DIRECTORY, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers/constants'
import { DIRECTORIES, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers/constants'
const staticRouter = express.Router()
// Cors is very important to let other servers access torrent and video files
staticRouter.use(cors())
// Videos path for webseed
// WebTorrent/Classic videos
staticRouter.use(
STATIC_PATHS.WEBSEED,
express.static(CONFIG.STORAGE.VIDEOS_DIR, { fallthrough: false }),
STATIC_PATHS.PRIVATE_WEBSEED,
optionalAuthenticate,
asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles),
express.static(DIRECTORIES.VIDEOS.PRIVATE, { fallthrough: false }),
handleStaticError
)
staticRouter.use(
STATIC_PATHS.WEBSEED,
express.static(DIRECTORIES.VIDEOS.PUBLIC, { fallthrough: false }),
handleStaticError
)
staticRouter.use(
STATIC_PATHS.REDUNDANCY,
express.static(CONFIG.STORAGE.REDUNDANCY_DIR, { fallthrough: false }),
@ -22,9 +36,16 @@ staticRouter.use(
)
// HLS
staticRouter.use(
STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS,
optionalAuthenticate,
asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles),
express.static(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, { fallthrough: false }),
handleStaticError
)
staticRouter.use(
STATIC_PATHS.STREAMING_PLAYLISTS.HLS,
express.static(HLS_STREAMING_PLAYLIST_DIRECTORY, { fallthrough: false }),
express.static(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, { fallthrough: false }),
handleStaticError
)

View File

@ -1,14 +1,15 @@
import { MutexInterface } from 'async-mutex'
import { Job } from 'bullmq'
import { FfmpegCommand } from 'fluent-ffmpeg'
import { readFile, writeFile } from 'fs-extra'
import { dirname } from 'path'
import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
import { pick } from '@shared/core-utils'
import { AvailableEncoders, VideoResolution } from '@shared/models'
import { logger, loggerTagsFactory } from '../logger'
import { getFFmpeg, runCommand } from './ffmpeg-commons'
import { presetCopy, presetOnlyAudio, presetVOD } from './ffmpeg-presets'
import { computeFPS, ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS } from './ffprobe-utils'
import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
const lTags = loggerTagsFactory('ffmpeg')
@ -22,6 +23,10 @@ interface BaseTranscodeVODOptions {
inputPath: string
outputPath: string
// Will be released after the ffmpeg started
// To prevent a bug where the input file does not exist anymore when running ffmpeg
inputFileMutexReleaser: MutexInterface.Releaser
availableEncoders: AvailableEncoders
profile: string
@ -94,6 +99,12 @@ async function transcodeVOD (options: TranscodeVODOptions) {
command = await builders[options.type](command, options)
command.on('start', () => {
setTimeout(() => {
options.inputFileMutexReleaser()
}, 1000)
})
await runCommand({ command, job: options.job })
await fixHLSPlaylistIfNeeded(options)

View File

@ -1,10 +1,10 @@
import { join } from 'path'
import { RESUMABLE_UPLOAD_DIRECTORY } from '../initializers/constants'
import { DIRECTORIES } from '@server/initializers/constants'
function getResumableUploadPath (filename?: string) {
if (filename) return join(RESUMABLE_UPLOAD_DIRECTORY, filename)
if (filename) return join(DIRECTORIES.RESUMABLE_UPLOAD, filename)
return RESUMABLE_UPLOAD_DIRECTORY
return DIRECTORIES.RESUMABLE_UPLOAD
}
// ---------------------------------------------------------------------------

View File

@ -164,7 +164,10 @@ function generateMagnetUri (
) {
const xs = videoFile.getTorrentUrl()
const announce = trackerUrls
let urlList = [ videoFile.getFileUrl(video) ]
let urlList = video.requiresAuth(video.uuid)
? []
: [ videoFile.getFileUrl(video) ]
const redundancies = videoFile.RedundancyVideos
if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl))
@ -240,6 +243,8 @@ function buildAnnounceList () {
}
function buildUrlList (video: MVideo, videoFile: MVideoFile) {
if (video.requiresAuth(video.uuid)) return []
return [ videoFile.getFileUrl(video) ]
}

View File

@ -662,10 +662,15 @@ const NSFW_POLICY_TYPES: { [ id: string ]: NSFWPolicyType } = {
// Express static paths (router)
const STATIC_PATHS = {
THUMBNAILS: '/static/thumbnails/',
WEBSEED: '/static/webseed/',
PRIVATE_WEBSEED: '/static/webseed/private/',
REDUNDANCY: '/static/redundancy/',
STREAMING_PLAYLISTS: {
HLS: '/static/streaming-playlists/hls'
HLS: '/static/streaming-playlists/hls',
PRIVATE_HLS: '/static/streaming-playlists/hls/private/'
}
}
const STATIC_DOWNLOAD_PATHS = {
@ -745,12 +750,32 @@ const LRU_CACHE = {
},
ACTOR_IMAGE_STATIC: {
MAX_SIZE: 500
},
STATIC_VIDEO_FILES_RIGHTS_CHECK: {
MAX_SIZE: 5000,
TTL: parseDurationToMs('10 seconds')
},
VIDEO_TOKENS: {
MAX_SIZE: 100_000,
TTL: parseDurationToMs('8 hours')
}
}
const RESUMABLE_UPLOAD_DIRECTORY = join(CONFIG.STORAGE.TMP_DIR, 'resumable-uploads')
const HLS_STREAMING_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls')
const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls')
const DIRECTORIES = {
RESUMABLE_UPLOAD: join(CONFIG.STORAGE.TMP_DIR, 'resumable-uploads'),
HLS_STREAMING_PLAYLIST: {
PUBLIC: join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls'),
PRIVATE: join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls', 'private')
},
VIDEOS: {
PUBLIC: CONFIG.STORAGE.VIDEOS_DIR,
PRIVATE: join(CONFIG.STORAGE.VIDEOS_DIR, 'private')
},
HLS_REDUNDANCY: join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls')
}
const RESUMABLE_UPLOAD_SESSION_LIFETIME = SCHEDULER_INTERVALS_MS.REMOVE_DANGLING_RESUMABLE_UPLOADS
@ -971,9 +996,8 @@ export {
PEERTUBE_VERSION,
LAZY_STATIC_PATHS,
SEARCH_INDEX,
RESUMABLE_UPLOAD_DIRECTORY,
DIRECTORIES,
RESUMABLE_UPLOAD_SESSION_LIFETIME,
HLS_REDUNDANCY_DIRECTORY,
P2P_MEDIA_LOADER_PEER_VERSION,
ACTOR_IMAGES_SIZE,
ACCEPT_HEADERS,
@ -1007,7 +1031,6 @@ export {
VIDEO_FILTERS,
ROUTE_CACHE_LIFETIME,
SORTABLE_COLUMNS,
HLS_STREAMING_PLAYLIST_DIRECTORY,
JOB_TTL,
DEFAULT_THEME_NAME,
NSFW_POLICY_TYPES,

View File

@ -10,7 +10,7 @@ import { ApplicationModel } from '../models/application/application'
import { OAuthClientModel } from '../models/oauth/oauth-client'
import { applicationExist, clientsExist, usersExist } from './checker-after-init'
import { CONFIG } from './config'
import { FILES_CACHE, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION, RESUMABLE_UPLOAD_DIRECTORY } from './constants'
import { DIRECTORIES, FILES_CACHE, LAST_MIGRATION_VERSION } from './constants'
import { sequelizeTypescript } from './database'
async function installApplication () {
@ -92,11 +92,13 @@ function createDirectoriesIfNotExist () {
tasks.push(ensureDir(dir))
}
// Playlist directories
tasks.push(ensureDir(HLS_STREAMING_PLAYLIST_DIRECTORY))
tasks.push(ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE))
tasks.push(ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC))
tasks.push(ensureDir(DIRECTORIES.VIDEOS.PUBLIC))
tasks.push(ensureDir(DIRECTORIES.VIDEOS.PRIVATE))
// Resumable upload directory
tasks.push(ensureDir(RESUMABLE_UPLOAD_DIRECTORY))
tasks.push(ensureDir(DIRECTORIES.RESUMABLE_UPLOAD))
return Promise.all(tasks)
}

View File

@ -95,14 +95,9 @@ async function handleOAuthToken (req: express.Request, options: { refreshTokenAu
function handleOAuthAuthenticate (
req: express.Request,
res: express.Response,
authenticateInQuery = false
res: express.Response
) {
const options = authenticateInQuery
? { allowBearerTokensInQueryString: true }
: {}
return oAuthServer.authenticate(new Request(req), new Response(res), options)
return oAuthServer.authenticate(new Request(req), new Response(res))
}
export {

View File

@ -82,7 +82,7 @@ async function loadStreamingPlaylistOrLog (streamingPlaylistId: number) {
async function loadFileOrLog (videoFileId: number) {
if (!videoFileId) return undefined
const file = await VideoFileModel.loadWithVideo(videoFileId)
const file = await VideoFileModel.load(videoFileId)
if (!file) {
logger.debug('Do not process torrent for file %d: does not exist anymore.', videoFileId)

View File

@ -3,10 +3,10 @@ import { remove } from 'fs-extra'
import { join } from 'path'
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { updateTorrentMetadata } from '@server/helpers/webtorrent'
import { CONFIG } from '@server/initializers/config'
import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants'
import { storeHLSFileFromFilename, storeWebTorrentFile } from '@server/lib/object-storage'
import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
import { VideoPathManager } from '@server/lib/video-path-manager'
import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state'
import { VideoModel } from '@server/models/video/video'
import { VideoJobInfoModel } from '@server/models/video/video-job-info'
@ -72,9 +72,9 @@ async function moveWebTorrentFiles (video: MVideoWithAllFiles) {
for (const file of video.VideoFiles) {
if (file.storage !== VideoStorage.FILE_SYSTEM) continue
const fileUrl = await storeWebTorrentFile(file.filename)
const fileUrl = await storeWebTorrentFile(video, file)
const oldPath = join(CONFIG.STORAGE.VIDEOS_DIR, file.filename)
const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file)
await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
}
}

View File

@ -18,6 +18,7 @@ import { VideoStreamingPlaylistModel } from '@server/models/video/video-streamin
import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@server/types/models'
import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models'
import { logger, loggerTagsFactory } from '../../../helpers/logger'
import { VideoPathManager } from '@server/lib/video-path-manager'
const lTags = loggerTagsFactory('live', 'job')
@ -205,18 +206,27 @@ async function assignReplayFilesToVideo (options: {
const concatenatedTsFiles = await readdir(replayDirectory)
for (const concatenatedTsFile of concatenatedTsFiles) {
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile)
const probe = await ffprobePromise(concatenatedTsFilePath)
const { audioStream } = await getAudioStream(concatenatedTsFilePath, probe)
const { resolution } = await getVideoStreamDimensionsInfo(concatenatedTsFilePath, probe)
await generateHlsPlaylistResolutionFromTS({
video,
concatenatedTsFilePath,
resolution,
isAAC: audioStream?.codec_name === 'aac'
})
try {
await generateHlsPlaylistResolutionFromTS({
video,
inputFileMutexReleaser,
concatenatedTsFilePath,
resolution,
isAAC: audioStream?.codec_name === 'aac'
})
} catch (err) {
logger.error('Cannot generate HLS playlist resolution from TS files.', { err })
}
inputFileMutexReleaser()
}
return video

View File

@ -94,15 +94,24 @@ async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, video: MV
const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist()
await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => {
return generateHlsPlaylistResolution({
video,
videoInputPath,
resolution: payload.resolution,
copyCodecs: payload.copyCodecs,
job
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
try {
await videoFileInput.getVideo().reload()
await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => {
return generateHlsPlaylistResolution({
video,
videoInputPath,
inputFileMutexReleaser,
resolution: payload.resolution,
copyCodecs: payload.copyCodecs,
job
})
})
})
} finally {
inputFileMutexReleaser()
}
logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid))
@ -177,38 +186,44 @@ async function onVideoFirstWebTorrentTranscoding (
transcodeType: TranscodeVODOptionsType,
user: MUserId
) {
const { resolution, audioStream } = await videoArg.probeMaxQualityFile()
const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid)
// Maybe the video changed in database, refresh it
const videoDatabase = await VideoModel.loadFull(videoArg.uuid)
// Video does not exist anymore
if (!videoDatabase) return undefined
try {
// Maybe the video changed in database, refresh it
const videoDatabase = await VideoModel.loadFull(videoArg.uuid)
// Video does not exist anymore
if (!videoDatabase) return undefined
// Generate HLS version of the original file
const originalFileHLSPayload = {
...payload,
const { resolution, audioStream } = await videoDatabase.probeMaxQualityFile()
hasAudio: !!audioStream,
resolution: videoDatabase.getMaxQualityFile().resolution,
// If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues
copyCodecs: transcodeType !== 'quick-transcode',
isMaxQuality: true
}
const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload)
const hasNewResolutions = await createLowerResolutionsJobs({
video: videoDatabase,
user,
videoFileResolution: resolution,
hasAudio: !!audioStream,
type: 'webtorrent',
isNewVideo: payload.isNewVideo ?? true
})
// Generate HLS version of the original file
const originalFileHLSPayload = {
...payload,
await VideoJobInfoModel.decrease(videoDatabase.uuid, 'pendingTranscode')
hasAudio: !!audioStream,
resolution: videoDatabase.getMaxQualityFile().resolution,
// If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues
copyCodecs: transcodeType !== 'quick-transcode',
isMaxQuality: true
}
const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload)
const hasNewResolutions = await createLowerResolutionsJobs({
video: videoDatabase,
user,
videoFileResolution: resolution,
hasAudio: !!audioStream,
type: 'webtorrent',
isNewVideo: payload.isNewVideo ?? true
})
// Move to next state if there are no other resolutions to generate
if (!hasHls && !hasNewResolutions) {
await retryTransactionWrapper(moveToNextState, { video: videoDatabase, isNewVideo: payload.isNewVideo })
await VideoJobInfoModel.decrease(videoDatabase.uuid, 'pendingTranscode')
// Move to next state if there are no other resolutions to generate
if (!hasHls && !hasNewResolutions) {
await retryTransactionWrapper(moveToNextState, { video: videoDatabase, isNewVideo: payload.isNewVideo })
}
} finally {
mutexReleaser()
}
}

View File

@ -1,8 +1,9 @@
import { basename, join } from 'path'
import { logger } from '@server/helpers/logger'
import { CONFIG } from '@server/initializers/config'
import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models'
import { MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/types/models'
import { getHLSDirectory } from '../paths'
import { VideoPathManager } from '../video-path-manager'
import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys'
import { listKeysOfPrefix, lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared'
@ -30,10 +31,10 @@ function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string)
// ---------------------------------------------------------------------------
function storeWebTorrentFile (filename: string) {
function storeWebTorrentFile (video: MVideo, file: MVideoFile) {
return storeObject({
inputPath: join(CONFIG.STORAGE.VIDEOS_DIR, filename),
objectStorageKey: generateWebTorrentObjectStorageKey(filename),
inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file),
objectStorageKey: generateWebTorrentObjectStorageKey(file.filename),
bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS
})
}

View File

@ -1,9 +1,10 @@
import { join } from 'path'
import { CONFIG } from '@server/initializers/config'
import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY, VIDEO_LIVE } from '@server/initializers/constants'
import { DIRECTORIES, VIDEO_LIVE } from '@server/initializers/constants'
import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'
import { removeFragmentedMP4Ext } from '@shared/core-utils'
import { buildUUID } from '@shared/extra-utils'
import { isVideoInPrivateDirectory } from './video-privacy'
// ################## Video file name ##################
@ -17,20 +18,24 @@ function generateHLSVideoFilename (resolution: number) {
// ################## Streaming playlist ##################
function getLiveDirectory (video: MVideoUUID) {
function getLiveDirectory (video: MVideo) {
return getHLSDirectory(video)
}
function getLiveReplayBaseDirectory (video: MVideoUUID) {
function getLiveReplayBaseDirectory (video: MVideo) {
return join(getLiveDirectory(video), VIDEO_LIVE.REPLAY_DIRECTORY)
}
function getHLSDirectory (video: MVideoUUID) {
return join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
function getHLSDirectory (video: MVideo) {
if (isVideoInPrivateDirectory(video.privacy)) {
return join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, video.uuid)
}
return join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, video.uuid)
}
function getHLSRedundancyDirectory (video: MVideoUUID) {
return join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
return join(DIRECTORIES.HLS_REDUNDANCY, video.uuid)
}
function getHlsResolutionPlaylistFilename (videoFilename: string) {

View File

@ -1,11 +1,14 @@
import { VideoModel } from '@server/models/video/video'
import { MVideoFullLight } from '@server/types/models'
import { MScheduleVideoUpdate } from '@server/types/models'
import { VideoPrivacy, VideoState } from '@shared/models'
import { logger } from '../../helpers/logger'
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
import { sequelizeTypescript } from '../../initializers/database'
import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update'
import { federateVideoIfNeeded } from '../activitypub/videos'
import { Notifier } from '../notifier'
import { addVideoJobsAfterUpdate } from '../video'
import { VideoPathManager } from '../video-path-manager'
import { setVideoPrivacy } from '../video-privacy'
import { AbstractScheduler } from './abstract-scheduler'
export class UpdateVideosScheduler extends AbstractScheduler {
@ -26,35 +29,54 @@ export class UpdateVideosScheduler extends AbstractScheduler {
if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined
const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate()
const publishedVideos: MVideoFullLight[] = []
for (const schedule of schedules) {
await sequelizeTypescript.transaction(async t => {
const video = await VideoModel.loadFull(schedule.videoId, t)
const videoOnly = await VideoModel.load(schedule.videoId)
const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoOnly.uuid)
logger.info('Executing scheduled video update on %s.', video.uuid)
try {
const { video, published } = await this.updateAVideo(schedule)
if (schedule.privacy) {
const wasConfidentialVideo = video.isConfidential()
const isNewVideo = video.isNewVideo(schedule.privacy)
if (published) Notifier.Instance.notifyOnVideoPublishedAfterScheduledUpdate(video)
} catch (err) {
logger.error('Cannot update video', { err })
}
video.setPrivacy(schedule.privacy)
await video.save({ transaction: t })
await federateVideoIfNeeded(video, isNewVideo, t)
mutexReleaser()
}
}
if (wasConfidentialVideo) {
publishedVideos.push(video)
}
private async updateAVideo (schedule: MScheduleVideoUpdate) {
let oldPrivacy: VideoPrivacy
let isNewVideo: boolean
let published = false
const video = await sequelizeTypescript.transaction(async t => {
const video = await VideoModel.loadFull(schedule.videoId, t)
if (video.state === VideoState.TO_TRANSCODE) return
logger.info('Executing scheduled video update on %s.', video.uuid)
if (schedule.privacy) {
isNewVideo = video.isNewVideo(schedule.privacy)
oldPrivacy = video.privacy
setVideoPrivacy(video, schedule.privacy)
await video.save({ transaction: t })
if (oldPrivacy === VideoPrivacy.PRIVATE) {
published = true
}
}
await schedule.destroy({ transaction: t })
})
}
await schedule.destroy({ transaction: t })
for (const v of publishedVideos) {
Notifier.Instance.notifyOnNewVideoIfNeeded(v)
Notifier.Instance.notifyOnVideoPublishedAfterScheduledUpdate(v)
}
return video
})
await addVideoJobsAfterUpdate({ video, oldPrivacy, isNewVideo, nameChanged: false })
return { video, published }
}
static get Instance () {

View File

@ -16,7 +16,7 @@ import { VideosRedundancyStrategy } from '../../../shared/models/redundancy'
import { logger, loggerTagsFactory } from '../../helpers/logger'
import { downloadWebTorrentVideo } from '../../helpers/webtorrent'
import { CONFIG } from '../../initializers/config'
import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants'
import { DIRECTORIES, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants'
import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url'
@ -262,7 +262,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy, lTags(video.uuid))
const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
const destDirectory = join(DIRECTORIES.HLS_REDUNDANCY, video.uuid)
const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video)
const maxSizeKB = this.getTotalFileSizes([], [ playlist ]) / 1000

View File

@ -1,3 +1,4 @@
import { MutexInterface } from 'async-mutex'
import { Job } from 'bullmq'
import { copyFile, ensureDir, move, remove, stat } from 'fs-extra'
import { basename, extname as extnameUtil, join } from 'path'
@ -6,11 +7,13 @@ import { retryTransactionWrapper } from '@server/helpers/database-utils'
import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
import { sequelizeTypescript } from '@server/initializers/database'
import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
import { pick } from '@shared/core-utils'
import { VideoResolution, VideoStorage } from '../../../shared/models/videos'
import {
buildFileMetadata,
canDoQuickTranscode,
computeResolutionsToTranscode,
ffprobePromise,
getVideoStreamDuration,
getVideoStreamFPS,
transcodeVOD,
@ -33,7 +36,7 @@ import { VideoTranscodingProfilesManager } from './default-transcoding-profiles'
*/
// Optimize the original video file and replace it. The resolution is not changed.
function optimizeOriginalVideofile (options: {
async function optimizeOriginalVideofile (options: {
video: MVideoFullLight
inputVideoFile: MVideoFile
job: Job
@ -43,49 +46,61 @@ function optimizeOriginalVideofile (options: {
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
const newExtname = '.mp4'
return VideoPathManager.Instance.makeAvailableVideoFile(inputVideoFile.withVideoOrPlaylist(video), async videoInputPath => {
const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
const transcodeType: TranscodeVODOptionsType = await canDoQuickTranscode(videoInputPath)
? 'quick-transcode'
: 'video'
try {
await video.reload()
const resolution = buildOriginalFileResolution(inputVideoFile.resolution)
const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video)
const transcodeOptions: TranscodeVODOptions = {
type: transcodeType,
const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async videoInputPath => {
const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
inputPath: videoInputPath,
outputPath: videoTranscodedPath,
const transcodeType: TranscodeVODOptionsType = await canDoQuickTranscode(videoInputPath)
? 'quick-transcode'
: 'video'
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
profile: CONFIG.TRANSCODING.PROFILE,
const resolution = buildOriginalFileResolution(inputVideoFile.resolution)
resolution,
const transcodeOptions: TranscodeVODOptions = {
type: transcodeType,
job
}
inputPath: videoInputPath,
outputPath: videoTranscodedPath,
// Could be very long!
await transcodeVOD(transcodeOptions)
inputFileMutexReleaser,
// Important to do this before getVideoFilename() to take in account the new filename
inputVideoFile.resolution = resolution
inputVideoFile.extname = newExtname
inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname)
inputVideoFile.storage = VideoStorage.FILE_SYSTEM
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
profile: CONFIG.TRANSCODING.PROFILE,
const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile)
resolution,
const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath)
await remove(videoInputPath)
job
}
return { transcodeType, videoFile }
})
// Could be very long!
await transcodeVOD(transcodeOptions)
// Important to do this before getVideoFilename() to take in account the new filename
inputVideoFile.resolution = resolution
inputVideoFile.extname = newExtname
inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname)
inputVideoFile.storage = VideoStorage.FILE_SYSTEM
const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile)
await remove(videoInputPath)
return { transcodeType, videoFile }
})
return result
} finally {
inputFileMutexReleaser()
}
}
// Transcode the original video file to a lower resolution compatible with WebTorrent
function transcodeNewWebTorrentResolution (options: {
async function transcodeNewWebTorrentResolution (options: {
video: MVideoFullLight
resolution: VideoResolution
job: Job
@ -95,53 +110,68 @@ function transcodeNewWebTorrentResolution (options: {
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
const newExtname = '.mp4'
return VideoPathManager.Instance.makeAvailableVideoFile(video.getMaxQualityFile().withVideoOrPlaylist(video), async videoInputPath => {
const newVideoFile = new VideoFileModel({
resolution,
extname: newExtname,
filename: generateWebTorrentVideoFilename(resolution, newExtname),
size: 0,
videoId: video.id
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
try {
await video.reload()
const file = video.getMaxQualityFile().withVideoOrPlaylist(video)
const result = await VideoPathManager.Instance.makeAvailableVideoFile(file, async videoInputPath => {
const newVideoFile = new VideoFileModel({
resolution,
extname: newExtname,
filename: generateWebTorrentVideoFilename(resolution, newExtname),
size: 0,
videoId: video.id
})
const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename)
const transcodeOptions = resolution === VideoResolution.H_NOVIDEO
? {
type: 'only-audio' as 'only-audio',
inputPath: videoInputPath,
outputPath: videoTranscodedPath,
inputFileMutexReleaser,
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
profile: CONFIG.TRANSCODING.PROFILE,
resolution,
job
}
: {
type: 'video' as 'video',
inputPath: videoInputPath,
outputPath: videoTranscodedPath,
inputFileMutexReleaser,
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
profile: CONFIG.TRANSCODING.PROFILE,
resolution,
job
}
await transcodeVOD(transcodeOptions)
return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, newVideoFile)
})
const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile)
const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename)
const transcodeOptions = resolution === VideoResolution.H_NOVIDEO
? {
type: 'only-audio' as 'only-audio',
inputPath: videoInputPath,
outputPath: videoTranscodedPath,
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
profile: CONFIG.TRANSCODING.PROFILE,
resolution,
job
}
: {
type: 'video' as 'video',
inputPath: videoInputPath,
outputPath: videoTranscodedPath,
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
profile: CONFIG.TRANSCODING.PROFILE,
resolution,
job
}
await transcodeVOD(transcodeOptions)
return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath)
})
return result
} finally {
inputFileMutexReleaser()
}
}
// Merge an image with an audio file to create a video
function mergeAudioVideofile (options: {
async function mergeAudioVideofile (options: {
video: MVideoFullLight
resolution: VideoResolution
job: Job
@ -151,54 +181,67 @@ function mergeAudioVideofile (options: {
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
const newExtname = '.mp4'
const inputVideoFile = video.getMinQualityFile()
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
return VideoPathManager.Instance.makeAvailableVideoFile(inputVideoFile.withVideoOrPlaylist(video), async audioInputPath => {
const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
try {
await video.reload()
// If the user updates the video preview during transcoding
const previewPath = video.getPreview().getPath()
const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath))
await copyFile(previewPath, tmpPreviewPath)
const inputVideoFile = video.getMinQualityFile()
const transcodeOptions = {
type: 'merge-audio' as 'merge-audio',
const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video)
inputPath: tmpPreviewPath,
outputPath: videoTranscodedPath,
const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async audioInputPath => {
const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
profile: CONFIG.TRANSCODING.PROFILE,
// If the user updates the video preview during transcoding
const previewPath = video.getPreview().getPath()
const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath))
await copyFile(previewPath, tmpPreviewPath)
audioPath: audioInputPath,
resolution,
const transcodeOptions = {
type: 'merge-audio' as 'merge-audio',
job
}
inputPath: tmpPreviewPath,
outputPath: videoTranscodedPath,
try {
await transcodeVOD(transcodeOptions)
inputFileMutexReleaser,
await remove(audioInputPath)
await remove(tmpPreviewPath)
} catch (err) {
await remove(tmpPreviewPath)
throw err
}
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
profile: CONFIG.TRANSCODING.PROFILE,
// Important to do this before getVideoFilename() to take in account the new file extension
inputVideoFile.extname = newExtname
inputVideoFile.resolution = resolution
inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname)
audioPath: audioInputPath,
resolution,
const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile)
// ffmpeg generated a new video file, so update the video duration
// See https://trac.ffmpeg.org/ticket/5456
video.duration = await getVideoStreamDuration(videoTranscodedPath)
await video.save()
job
}
return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath)
})
try {
await transcodeVOD(transcodeOptions)
await remove(audioInputPath)
await remove(tmpPreviewPath)
} catch (err) {
await remove(tmpPreviewPath)
throw err
}
// Important to do this before getVideoFilename() to take in account the new file extension
inputVideoFile.extname = newExtname
inputVideoFile.resolution = resolution
inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname)
// ffmpeg generated a new video file, so update the video duration
// See https://trac.ffmpeg.org/ticket/5456
video.duration = await getVideoStreamDuration(videoTranscodedPath)
await video.save()
return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile)
})
return result
} finally {
inputFileMutexReleaser()
}
}
// Concat TS segments from a live video to a fragmented mp4 HLS playlist
@ -207,13 +250,13 @@ async function generateHlsPlaylistResolutionFromTS (options: {
concatenatedTsFilePath: string
resolution: VideoResolution
isAAC: boolean
inputFileMutexReleaser: MutexInterface.Releaser
}) {
return generateHlsPlaylistCommon({
video: options.video,
resolution: options.resolution,
inputPath: options.concatenatedTsFilePath,
type: 'hls-from-ts' as 'hls-from-ts',
isAAC: options.isAAC
inputPath: options.concatenatedTsFilePath,
...pick(options, [ 'video', 'resolution', 'inputFileMutexReleaser', 'isAAC' ])
})
}
@ -223,15 +266,14 @@ function generateHlsPlaylistResolution (options: {
videoInputPath: string
resolution: VideoResolution
copyCodecs: boolean
inputFileMutexReleaser: MutexInterface.Releaser
job?: Job
}) {
return generateHlsPlaylistCommon({
video: options.video,
resolution: options.resolution,
copyCodecs: options.copyCodecs,
inputPath: options.videoInputPath,
type: 'hls' as 'hls',
job: options.job
inputPath: options.videoInputPath,
...pick(options, [ 'video', 'resolution', 'copyCodecs', 'inputFileMutexReleaser', 'job' ])
})
}
@ -251,27 +293,39 @@ async function onWebTorrentVideoFileTranscoding (
video: MVideoFullLight,
videoFile: MVideoFile,
transcodingPath: string,
outputPath: string
newVideoFile: MVideoFile
) {
const stats = await stat(transcodingPath)
const fps = await getVideoStreamFPS(transcodingPath)
const metadata = await buildFileMetadata(transcodingPath)
const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
await move(transcodingPath, outputPath, { overwrite: true })
try {
await video.reload()
videoFile.size = stats.size
videoFile.fps = fps
videoFile.metadata = metadata
const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile)
await createTorrentAndSetInfoHash(video, videoFile)
const stats = await stat(transcodingPath)
const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution })
if (oldFile) await video.removeWebTorrentFile(oldFile)
const probe = await ffprobePromise(transcodingPath)
const fps = await getVideoStreamFPS(transcodingPath, probe)
const metadata = await buildFileMetadata(transcodingPath, probe)
await VideoFileModel.customUpsert(videoFile, 'video', undefined)
video.VideoFiles = await video.$get('VideoFiles')
await move(transcodingPath, outputPath, { overwrite: true })
return { video, videoFile }
videoFile.size = stats.size
videoFile.fps = fps
videoFile.metadata = metadata
await createTorrentAndSetInfoHash(video, videoFile)
const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution })
if (oldFile) await video.removeWebTorrentFile(oldFile)
await VideoFileModel.customUpsert(videoFile, 'video', undefined)
video.VideoFiles = await video.$get('VideoFiles')
return { video, videoFile }
} finally {
mutexReleaser()
}
}
async function generateHlsPlaylistCommon (options: {
@ -279,12 +333,15 @@ async function generateHlsPlaylistCommon (options: {
video: MVideo
inputPath: string
resolution: VideoResolution
inputFileMutexReleaser: MutexInterface.Releaser
copyCodecs?: boolean
isAAC?: boolean
job?: Job
}) {
const { type, video, inputPath, resolution, copyCodecs, isAAC, job } = options
const { type, video, inputPath, resolution, copyCodecs, isAAC, job, inputFileMutexReleaser } = options
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
const videoTranscodedBasePath = join(transcodeDirectory, type)
@ -308,6 +365,8 @@ async function generateHlsPlaylistCommon (options: {
isAAC,
inputFileMutexReleaser,
hlsPlaylist: {
videoFilename
},
@ -333,40 +392,54 @@ async function generateHlsPlaylistCommon (options: {
videoStreamingPlaylistId: playlist.id
})
const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile)
await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video))
const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
// Move playlist file
const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename)
await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true })
// Move video file
await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true })
try {
// VOD transcoding is a long task, refresh video attributes
await video.reload()
// Update video duration if it was not set (in case of a live for example)
if (!video.duration) {
video.duration = await getVideoStreamDuration(videoFilePath)
await video.save()
const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile)
await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video))
// Move playlist file
const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename)
await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true })
// Move video file
await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true })
// Update video duration if it was not set (in case of a live for example)
if (!video.duration) {
video.duration = await getVideoStreamDuration(videoFilePath)
await video.save()
}
const stats = await stat(videoFilePath)
newVideoFile.size = stats.size
newVideoFile.fps = await getVideoStreamFPS(videoFilePath)
newVideoFile.metadata = await buildFileMetadata(videoFilePath)
await createTorrentAndSetInfoHash(playlist, newVideoFile)
const oldFile = await VideoFileModel.loadHLSFile({
playlistId: playlist.id,
fps: newVideoFile.fps,
resolution: newVideoFile.resolution
})
if (oldFile) {
await video.removeStreamingPlaylistVideoFile(playlist, oldFile)
await oldFile.destroy()
}
const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)
await updatePlaylistAfterFileChange(video, playlist)
return { resolutionPlaylistPath, videoFile: savedVideoFile }
} finally {
mutexReleaser()
}
const stats = await stat(videoFilePath)
newVideoFile.size = stats.size
newVideoFile.fps = await getVideoStreamFPS(videoFilePath)
newVideoFile.metadata = await buildFileMetadata(videoFilePath)
await createTorrentAndSetInfoHash(playlist, newVideoFile)
const oldFile = await VideoFileModel.loadHLSFile({ playlistId: playlist.id, fps: newVideoFile.fps, resolution: newVideoFile.resolution })
if (oldFile) {
await video.removeStreamingPlaylistVideoFile(playlist, oldFile)
await oldFile.destroy()
}
const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)
await updatePlaylistAfterFileChange(video, playlist)
return { resolutionPlaylistPath, videoFile: savedVideoFile }
}
function buildOriginalFileResolution (inputResolution: number) {

View File

@ -1,29 +1,31 @@
import { Mutex } from 'async-mutex'
import { remove } from 'fs-extra'
import { extname, join } from 'path'
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { extractVideo } from '@server/helpers/video'
import { CONFIG } from '@server/initializers/config'
import {
MStreamingPlaylistVideo,
MVideo,
MVideoFile,
MVideoFileStreamingPlaylistVideo,
MVideoFileVideo,
MVideoUUID
} from '@server/types/models'
import { DIRECTORIES } from '@server/initializers/constants'
import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '@server/types/models'
import { buildUUID } from '@shared/extra-utils'
import { VideoStorage } from '@shared/models'
import { makeHLSFileAvailable, makeWebTorrentFileAvailable } from './object-storage'
import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths'
import { isVideoInPrivateDirectory } from './video-privacy'
type MakeAvailableCB <T> = (path: string) => Promise<T> | T
const lTags = loggerTagsFactory('video-path-manager')
class VideoPathManager {
private static instance: VideoPathManager
// Key is a video UUID
private readonly videoFileMutexStore = new Map<string, Mutex>()
private constructor () {}
getFSHLSOutputPath (video: MVideoUUID, filename?: string) {
getFSHLSOutputPath (video: MVideo, filename?: string) {
const base = getHLSDirectory(video)
if (!filename) return base
@ -41,13 +43,17 @@ class VideoPathManager {
}
getFSVideoFileOutputPath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
if (videoFile.isHLS()) {
const video = extractVideo(videoOrPlaylist)
const video = extractVideo(videoOrPlaylist)
if (videoFile.isHLS()) {
return join(getHLSDirectory(video), videoFile.filename)
}
return join(CONFIG.STORAGE.VIDEOS_DIR, videoFile.filename)
if (isVideoInPrivateDirectory(video.privacy)) {
return join(DIRECTORIES.VIDEOS.PRIVATE, videoFile.filename)
}
return join(DIRECTORIES.VIDEOS.PUBLIC, videoFile.filename)
}
async makeAvailableVideoFile <T> (videoFile: MVideoFileVideo | MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB<T>) {
@ -113,6 +119,27 @@ class VideoPathManager {
)
}
async lockFiles (videoUUID: string) {
if (!this.videoFileMutexStore.has(videoUUID)) {
this.videoFileMutexStore.set(videoUUID, new Mutex())
}
const mutex = this.videoFileMutexStore.get(videoUUID)
const releaser = await mutex.acquire()
logger.debug('Locked files of %s.', videoUUID, lTags(videoUUID))
return releaser
}
unlockFiles (videoUUID: string) {
const mutex = this.videoFileMutexStore.get(videoUUID)
mutex.release()
logger.debug('Released lockfiles of %s.', videoUUID, lTags(videoUUID))
}
private async makeAvailableFactory <T> (method: () => Promise<string> | string, clean: boolean, cb: MakeAvailableCB<T>) {
let result: T

View File

@ -0,0 +1,96 @@
import { move } from 'fs-extra'
import { join } from 'path'
import { logger } from '@server/helpers/logger'
import { DIRECTORIES } from '@server/initializers/constants'
import { MVideo, MVideoFullLight } from '@server/types/models'
import { VideoPrivacy } from '@shared/models'
function setVideoPrivacy (video: MVideo, newPrivacy: VideoPrivacy) {
if (video.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) {
video.publishedAt = new Date()
}
video.privacy = newPrivacy
}
function isVideoInPrivateDirectory (privacy: VideoPrivacy) {
return privacy === VideoPrivacy.PRIVATE || privacy === VideoPrivacy.INTERNAL
}
function isVideoInPublicDirectory (privacy: VideoPrivacy) {
return !isVideoInPrivateDirectory(privacy)
}
async function moveFilesIfPrivacyChanged (video: MVideoFullLight, oldPrivacy: VideoPrivacy) {
// Now public, previously private
if (isVideoInPublicDirectory(video.privacy) && isVideoInPrivateDirectory(oldPrivacy)) {
await moveFiles({ type: 'private-to-public', video })
return true
}
// Now private, previously public
if (isVideoInPrivateDirectory(video.privacy) && isVideoInPublicDirectory(oldPrivacy)) {
await moveFiles({ type: 'public-to-private', video })
return true
}
return false
}
export {
setVideoPrivacy,
isVideoInPrivateDirectory,
isVideoInPublicDirectory,
moveFilesIfPrivacyChanged
}
// ---------------------------------------------------------------------------
async function moveFiles (options: {
type: 'private-to-public' | 'public-to-private'
video: MVideoFullLight
}) {
const { type, video } = options
const directories = type === 'private-to-public'
? {
webtorrent: { old: DIRECTORIES.VIDEOS.PRIVATE, new: DIRECTORIES.VIDEOS.PUBLIC },
hls: { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC }
}
: {
webtorrent: { old: DIRECTORIES.VIDEOS.PUBLIC, new: DIRECTORIES.VIDEOS.PRIVATE },
hls: { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE }
}
for (const file of video.VideoFiles) {
const source = join(directories.webtorrent.old, file.filename)
const destination = join(directories.webtorrent.new, file.filename)
try {
logger.info('Moving WebTorrent files of %s after privacy change (%s -> %s).', video.uuid, source, destination)
await move(source, destination)
} catch (err) {
logger.error('Cannot move webtorrent file %s to %s after privacy change', source, destination, { err })
}
}
const hls = video.getHLSPlaylist()
if (hls) {
const source = join(directories.hls.old, video.uuid)
const destination = join(directories.hls.new, video.uuid)
try {
logger.info('Moving HLS files of %s after privacy change (%s -> %s).', video.uuid, source, destination)
await move(source, destination)
} catch (err) {
logger.error('Cannot move HLS file %s to %s after privacy change', source, destination, { err })
}
}
}

View File

@ -0,0 +1,49 @@
import LRUCache from 'lru-cache'
import { LRU_CACHE } from '@server/initializers/constants'
import { buildUUID } from '@shared/extra-utils'
// ---------------------------------------------------------------------------
// Create temporary tokens that can be used as URL query parameters to access video static files
// ---------------------------------------------------------------------------
class VideoTokensManager {
private static instance: VideoTokensManager
private readonly lruCache = new LRUCache<string, string>({
max: LRU_CACHE.VIDEO_TOKENS.MAX_SIZE,
ttl: LRU_CACHE.VIDEO_TOKENS.TTL
})
private constructor () {}
create (videoUUID: string) {
const token = buildUUID()
const expires = new Date(new Date().getTime() + LRU_CACHE.VIDEO_TOKENS.TTL)
this.lruCache.set(token, videoUUID)
return { token, expires }
}
hasToken (options: {
token: string
videoUUID: string
}) {
const value = this.lruCache.get(options.token)
if (!value) return false
return value === options.videoUUID
}
static get Instance () {
return this.instance || (this.instance = new this())
}
}
// ---------------------------------------------------------------------------
export {
VideoTokensManager
}

View File

@ -7,10 +7,11 @@ import { TagModel } from '@server/models/video/tag'
import { VideoModel } from '@server/models/video/video'
import { VideoJobInfoModel } from '@server/models/video/video-job-info'
import { FilteredModelAttributes } from '@server/types'
import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
import { ThumbnailType, VideoCreate, VideoPrivacy, VideoState, VideoTranscodingPayload } from '@shared/models'
import { CreateJobOptions } from './job-queue/job-queue'
import { MThumbnail, MUserId, MVideoFile, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
import { ManageVideoTorrentPayload, ThumbnailType, VideoCreate, VideoPrivacy, VideoState, VideoTranscodingPayload } from '@shared/models'
import { CreateJobArgument, CreateJobOptions, JobQueue } from './job-queue/job-queue'
import { updateVideoMiniatureFromExisting } from './thumbnail'
import { moveFilesIfPrivacyChanged } from './video-privacy'
function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> {
return {
@ -177,6 +178,59 @@ const getCachedVideoDuration = memoizee(getVideoDuration, {
// ---------------------------------------------------------------------------
async function addVideoJobsAfterUpdate (options: {
video: MVideoFullLight
isNewVideo: boolean
nameChanged: boolean
oldPrivacy: VideoPrivacy
}) {
const { video, nameChanged, oldPrivacy, isNewVideo } = options
const jobs: CreateJobArgument[] = []
const filePathChanged = await moveFilesIfPrivacyChanged(video, oldPrivacy)
if (!video.isLive && (nameChanged || filePathChanged)) {
for (const file of (video.VideoFiles || [])) {
const payload: ManageVideoTorrentPayload = { action: 'update-metadata', videoId: video.id, videoFileId: file.id }
jobs.push({ type: 'manage-video-torrent', payload })
}
const hls = video.getHLSPlaylist()
for (const file of (hls?.VideoFiles || [])) {
const payload: ManageVideoTorrentPayload = { action: 'update-metadata', streamingPlaylistId: hls.id, videoFileId: file.id }
jobs.push({ type: 'manage-video-torrent', payload })
}
}
jobs.push({
type: 'federate-video',
payload: {
videoUUID: video.uuid,
isNewVideo
}
})
const wasConfidentialVideo = new Set([ VideoPrivacy.PRIVATE, VideoPrivacy.UNLISTED, VideoPrivacy.INTERNAL ]).has(oldPrivacy)
if (wasConfidentialVideo) {
jobs.push({
type: 'notify',
payload: {
action: 'new-video',
videoUUID: video.uuid
}
})
}
return JobQueue.Instance.createSequentialJobFlow(...jobs)
}
// ---------------------------------------------------------------------------
export {
buildLocalVideoFromReq,
buildVideoThumbnailsFromReq,
@ -185,5 +239,6 @@ export {
buildTranscodingJob,
buildMoveToObjectStorageJob,
getTranscodingJobPriority,
addVideoJobsAfterUpdate,
getCachedVideoDuration
}

View File

@ -5,8 +5,8 @@ import { HttpStatusCode } from '../../shared/models/http/http-error-codes'
import { logger } from '../helpers/logger'
import { handleOAuthAuthenticate } from '../lib/auth/oauth'
function authenticate (req: express.Request, res: express.Response, next: express.NextFunction, authenticateInQuery = false) {
handleOAuthAuthenticate(req, res, authenticateInQuery)
function authenticate (req: express.Request, res: express.Response, next: express.NextFunction) {
handleOAuthAuthenticate(req, res)
.then((token: any) => {
res.locals.oauth = { token }
res.locals.authenticated = true
@ -47,7 +47,7 @@ function authenticateSocket (socket: Socket, next: (err?: any) => void) {
.catch(err => logger.error('Cannot get access token.', { err }))
}
function authenticatePromise (req: express.Request, res: express.Response, authenticateInQuery = false) {
function authenticatePromise (req: express.Request, res: express.Response) {
return new Promise<void>(resolve => {
// Already authenticated? (or tried to)
if (res.locals.oauth?.token.User) return resolve()
@ -59,7 +59,7 @@ function authenticatePromise (req: express.Request, res: express.Response, authe
})
}
authenticate(req, res, () => resolve(), authenticateInQuery)
authenticate(req, res, () => resolve())
})
}

View File

@ -1,7 +1,6 @@
export * from './activitypub'
export * from './videos'
export * from './abuse'
export * from './account'
export * from './activitypub'
export * from './actor-image'
export * from './blocklist'
export * from './bulk'
@ -10,8 +9,8 @@ export * from './express'
export * from './feeds'
export * from './follows'
export * from './jobs'
export * from './metrics'
export * from './logs'
export * from './metrics'
export * from './oembed'
export * from './pagination'
export * from './plugins'
@ -19,9 +18,11 @@ export * from './redundancy'
export * from './search'
export * from './server'
export * from './sort'
export * from './static'
export * from './themes'
export * from './user-history'
export * from './user-notifications'
export * from './user-subscriptions'
export * from './users'
export * from './videos'
export * from './webfinger'

View File

@ -1,7 +1,7 @@
import { Request, Response } from 'express'
import { isUUIDValid } from '@server/helpers/custom-validators/misc'
import { loadVideo, VideoLoadType } from '@server/lib/model-loaders'
import { isAbleToUploadVideo } from '@server/lib/user'
import { VideoTokensManager } from '@server/lib/video-tokens-manager'
import { authenticatePromise } from '@server/middlewares/auth'
import { VideoModel } from '@server/models/video/video'
import { VideoChannelModel } from '@server/models/video/video-channel'
@ -108,26 +108,21 @@ async function checkCanSeeVideo (options: {
res: Response
paramId: string
video: MVideo
authenticateInQuery?: boolean // default false
}) {
const { req, res, video, paramId, authenticateInQuery = false } = options
const { req, res, video, paramId } = options
if (video.requiresAuth()) {
return checkCanSeeAuthVideo(req, res, video, authenticateInQuery)
if (video.requiresAuth(paramId)) {
return checkCanSeeAuthVideo(req, res, video)
}
if (video.privacy === VideoPrivacy.UNLISTED) {
if (isUUIDValid(paramId)) return true
return checkCanSeeAuthVideo(req, res, video, authenticateInQuery)
if (video.privacy === VideoPrivacy.UNLISTED || video.privacy === VideoPrivacy.PUBLIC) {
return true
}
if (video.privacy === VideoPrivacy.PUBLIC) return true
throw new Error('Fatal error when checking video right ' + video.url)
throw new Error('Unknown video privacy when checking video right ' + video.url)
}
async function checkCanSeeAuthVideo (req: Request, res: Response, video: MVideoId | MVideoWithRights, authenticateInQuery = false) {
async function checkCanSeeAuthVideo (req: Request, res: Response, video: MVideoId | MVideoWithRights) {
const fail = () => {
res.fail({
status: HttpStatusCode.FORBIDDEN_403,
@ -137,7 +132,7 @@ async function checkCanSeeAuthVideo (req: Request, res: Response, video: MVideoI
return false
}
await authenticatePromise(req, res, authenticateInQuery)
await authenticatePromise(req, res)
const user = res.locals.oauth?.token.User
if (!user) return fail()
@ -173,6 +168,36 @@ async function checkCanSeeAuthVideo (req: Request, res: Response, video: MVideoI
// ---------------------------------------------------------------------------
async function checkCanAccessVideoStaticFiles (options: {
video: MVideo
req: Request
res: Response
paramId: string
}) {
const { video, req, res, paramId } = options
if (res.locals.oauth?.token.User) {
return checkCanSeeVideo(options)
}
if (!video.requiresAuth(paramId)) return true
const videoFileToken = req.query.videoFileToken
if (!videoFileToken) {
res.sendStatus(HttpStatusCode.FORBIDDEN_403)
return false
}
if (VideoTokensManager.Instance.hasToken({ token: videoFileToken, videoUUID: video.uuid })) {
return true
}
res.sendStatus(HttpStatusCode.FORBIDDEN_403)
return false
}
// ---------------------------------------------------------------------------
function checkUserCanManageVideo (user: MUser, video: MVideoAccountLight, right: UserRight, res: Response, onlyOwned = true) {
// Retrieve the user who did the request
if (onlyOwned && video.isOwned() === false) {
@ -220,6 +245,7 @@ export {
doesVideoExist,
doesVideoFileOfVideoExist,
checkCanAccessVideoStaticFiles,
checkUserCanManageVideo,
checkCanSeeVideo,
checkUserQuota

View File

@ -0,0 +1,131 @@
import express from 'express'
import { query } from 'express-validator'
import LRUCache from 'lru-cache'
import { basename, dirname } from 'path'
import { exists, isUUIDValid } from '@server/helpers/custom-validators/misc'
import { logger } from '@server/helpers/logger'
import { LRU_CACHE } from '@server/initializers/constants'
import { VideoModel } from '@server/models/video/video'
import { VideoFileModel } from '@server/models/video/video-file'
import { HttpStatusCode } from '@shared/models'
import { areValidationErrors, checkCanAccessVideoStaticFiles } from './shared'
const staticFileTokenBypass = new LRUCache<string, boolean>({
max: LRU_CACHE.STATIC_VIDEO_FILES_RIGHTS_CHECK.MAX_SIZE,
ttl: LRU_CACHE.STATIC_VIDEO_FILES_RIGHTS_CHECK.TTL
})
const ensureCanAccessVideoPrivateWebTorrentFiles = [
query('videoFileToken').optional().custom(exists),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
if (areValidationErrors(req, res)) return
const token = extractTokenOrDie(req, res)
if (!token) return
const cacheKey = token + '-' + req.originalUrl
if (staticFileTokenBypass.has(cacheKey)) {
const allowedFromCache = staticFileTokenBypass.get(cacheKey)
if (allowedFromCache === true) return next()
return res.sendStatus(HttpStatusCode.FORBIDDEN_403)
}
const allowed = await isWebTorrentAllowed(req, res)
staticFileTokenBypass.set(cacheKey, allowed)
if (allowed !== true) return
return next()
}
]
const ensureCanAccessPrivateVideoHLSFiles = [
query('videoFileToken').optional().custom(exists),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
if (areValidationErrors(req, res)) return
const videoUUID = basename(dirname(req.originalUrl))
if (!isUUIDValid(videoUUID)) {
logger.debug('Path does not contain valid video UUID to serve static file %s', req.originalUrl)
return res.sendStatus(HttpStatusCode.FORBIDDEN_403)
}
const token = extractTokenOrDie(req, res)
if (!token) return
const cacheKey = token + '-' + videoUUID
if (staticFileTokenBypass.has(cacheKey)) {
const allowedFromCache = staticFileTokenBypass.get(cacheKey)
if (allowedFromCache === true) return next()
return res.sendStatus(HttpStatusCode.FORBIDDEN_403)
}
const allowed = await isHLSAllowed(req, res, videoUUID)
staticFileTokenBypass.set(cacheKey, allowed)
if (allowed !== true) return
return next()
}
]
export {
ensureCanAccessVideoPrivateWebTorrentFiles,
ensureCanAccessPrivateVideoHLSFiles
}
// ---------------------------------------------------------------------------
async function isWebTorrentAllowed (req: express.Request, res: express.Response) {
const filename = basename(req.path)
const file = await VideoFileModel.loadWithVideoByFilename(filename)
if (!file) {
logger.debug('Unknown static file %s to serve', req.originalUrl, { filename })
res.sendStatus(HttpStatusCode.FORBIDDEN_403)
return false
}
const video = file.getVideo()
return checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid })
}
async function isHLSAllowed (req: express.Request, res: express.Response, videoUUID: string) {
const video = await VideoModel.load(videoUUID)
if (!video) {
logger.debug('Unknown static file %s to serve', req.originalUrl, { videoUUID })
res.sendStatus(HttpStatusCode.FORBIDDEN_403)
return false
}
return checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid })
}
function extractTokenOrDie (req: express.Request, res: express.Response) {
const token = res.locals.oauth?.token.accessToken || req.query.videoFileToken
if (!token) {
return res.fail({
message: 'Bearer token is missing in headers or video file token is missing in URL query parameters',
status: HttpStatusCode.FORBIDDEN_403
})
}
return token
}

View File

@ -7,7 +7,7 @@ import { getServerActor } from '@server/models/application/application'
import { ExpressPromiseHandler } from '@server/types/express-handler'
import { MUserAccountId, MVideoFullLight } from '@server/types/models'
import { arrayify, getAllPrivacies } from '@shared/core-utils'
import { HttpStatusCode, ServerErrorCode, UserRight, VideoInclude } from '@shared/models'
import { HttpStatusCode, ServerErrorCode, UserRight, VideoInclude, VideoState } from '@shared/models'
import {
exists,
isBooleanValid,
@ -48,6 +48,7 @@ import { Hooks } from '../../../lib/plugins/hooks'
import { VideoModel } from '../../../models/video/video'
import {
areValidationErrors,
checkCanAccessVideoStaticFiles,
checkCanSeeVideo,
checkUserCanManageVideo,
checkUserQuota,
@ -232,6 +233,11 @@ const videosUpdateValidator = getCommonVideoEditAttributes().concat([
if (areErrorsInScheduleUpdate(req, res)) return cleanUpReqFiles(req)
if (!await doesVideoExist(req.params.id, res)) return cleanUpReqFiles(req)
const video = getVideoWithAttributes(res)
if (req.body.privacy && video.isLive && video.state !== VideoState.WAITING_FOR_LIVE) {
return res.fail({ message: 'Cannot update privacy of a live that has already started' })
}
// Check if the user who did the request is able to update the video
const user = res.locals.oauth.token.User
if (!checkUserCanManageVideo(user, res.locals.videoAll, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req)
@ -271,10 +277,7 @@ async function checkVideoFollowConstraints (req: express.Request, res: express.R
})
}
const videosCustomGetValidator = (
fetchType: 'for-api' | 'all' | 'only-video' | 'only-immutable-attributes',
authenticateInQuery = false
) => {
const videosCustomGetValidator = (fetchType: 'for-api' | 'all' | 'only-video' | 'only-immutable-attributes') => {
return [
isValidVideoIdParam('id'),
@ -287,7 +290,7 @@ const videosCustomGetValidator = (
const video = getVideoWithAttributes(res) as MVideoFullLight
if (!await checkCanSeeVideo({ req, res, video, paramId: req.params.id, authenticateInQuery })) return
if (!await checkCanSeeVideo({ req, res, video, paramId: req.params.id })) return
return next()
}
@ -295,7 +298,6 @@ const videosCustomGetValidator = (
}
const videosGetValidator = videosCustomGetValidator('all')
const videosDownloadValidator = videosCustomGetValidator('all', true)
const videoFileMetadataGetValidator = getCommonVideoEditAttributes().concat([
isValidVideoIdParam('id'),
@ -311,6 +313,21 @@ const videoFileMetadataGetValidator = getCommonVideoEditAttributes().concat([
}
])
const videosDownloadValidator = [
isValidVideoIdParam('id'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
if (areValidationErrors(req, res)) return
if (!await doesVideoExist(req.params.id, res, 'all')) return
const video = getVideoWithAttributes(res)
if (!await checkCanAccessVideoStaticFiles({ req, res, video, paramId: req.params.id })) return
return next()
}
]
const videosRemoveValidator = [
isValidVideoIdParam('id'),
@ -372,7 +389,7 @@ function getCommonVideoEditAttributes () {
.custom(isBooleanValid).withMessage('Should have a valid waitTranscoding boolean'),
body('privacy')
.optional()
.customSanitizer(toValueOrNull)
.customSanitizer(toIntOrNull)
.custom(isVideoPrivacyValid),
body('description')
.optional()

View File

@ -34,6 +34,7 @@ import {
import {
MServer,
MStreamingPlaylistRedundanciesOpt,
MUserId,
MVideo,
MVideoAP,
MVideoFile,
@ -245,8 +246,12 @@ function sortByResolutionDesc (fileA: MVideoFile, fileB: MVideoFile) {
function videoFilesModelToFormattedJSON (
video: MVideoFormattable,
videoFiles: MVideoFileRedundanciesOpt[],
includeMagnet = true
options: {
includeMagnet?: boolean // default true
} = {}
): VideoFile[] {
const { includeMagnet = true } = options
const trackerUrls = includeMagnet
? video.getTrackerUrls()
: []
@ -281,11 +286,14 @@ function videoFilesModelToFormattedJSON (
})
}
function addVideoFilesInAPAcc (
acc: ActivityUrlObject[] | ActivityTagObject[],
video: MVideo,
function addVideoFilesInAPAcc (options: {
acc: ActivityUrlObject[] | ActivityTagObject[]
video: MVideo
files: MVideoFile[]
) {
user?: MUserId
}) {
const { acc, video, files } = options
const trackerUrls = video.getTrackerUrls()
const sortedFiles = (files || [])
@ -370,7 +378,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
}
]
addVideoFilesInAPAcc(url, video, video.VideoFiles || [])
addVideoFilesInAPAcc({ acc: url, video, files: video.VideoFiles || [] })
for (const playlist of (video.VideoStreamingPlaylists || [])) {
const tag = playlist.p2pMediaLoaderInfohashes
@ -382,7 +390,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
href: playlist.getSha256SegmentsUrl(video)
})
addVideoFilesInAPAcc(tag, video, playlist.VideoFiles || [])
addVideoFilesInAPAcc({ acc: tag, video, files: playlist.VideoFiles || [] })
url.push({
type: 'Link',

View File

@ -24,6 +24,7 @@ import { extractVideo } from '@server/helpers/video'
import { buildRemoteVideoBaseUrl } from '@server/lib/activitypub/url'
import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage'
import { getFSTorrentFilePath } from '@server/lib/paths'
import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
import { VideoResolution, VideoStorage } from '@shared/models'
import { AttributesOnly } from '@shared/typescript-utils'
@ -295,6 +296,16 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
return VideoFileModel.findOne(query)
}
static loadWithVideoByFilename (filename: string): Promise<MVideoFileVideo | MVideoFileStreamingPlaylistVideo> {
const query = {
where: {
filename
}
}
return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
}
static loadWithVideoOrPlaylistByTorrentFilename (filename: string) {
const query = {
where: {
@ -305,6 +316,10 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
}
static load (id: number): Promise<MVideoFile> {
return VideoFileModel.findByPk(id)
}
static loadWithMetadata (id: number) {
return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id)
}
@ -467,7 +482,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
}
getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo {
if (this.videoId) return (this as MVideoFileVideo).Video
if (this.videoId || (this as MVideoFileVideo).Video) return (this as MVideoFileVideo).Video
return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist
}
@ -508,7 +523,17 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
}
getFileStaticPath (video: MVideo) {
if (this.isHLS()) return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename)
if (this.isHLS()) {
if (isVideoInPrivateDirectory(video.privacy)) {
return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.filename)
}
return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename)
}
if (isVideoInPrivateDirectory(video.privacy)) {
return join(STATIC_PATHS.PRIVATE_WEBSEED, this.filename)
}
return join(STATIC_PATHS.WEBSEED, this.filename)
}

View File

@ -17,6 +17,7 @@ import {
} from 'sequelize-typescript'
import { getHLSPublicFileUrl } from '@server/lib/object-storage'
import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '@server/lib/paths'
import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
import { VideoFileModel } from '@server/models/video/video-file'
import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models'
import { sha1 } from '@shared/extra-utils'
@ -250,7 +251,7 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
return getHLSPublicFileUrl(this.playlistUrl)
}
return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid)
return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video)
}
return this.playlistUrl
@ -262,7 +263,7 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
return getHLSPublicFileUrl(this.segmentsSha256Url)
}
return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid)
return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video)
}
return this.segmentsSha256Url
@ -287,11 +288,19 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
return Object.assign(this, { Video: video })
}
private getMasterPlaylistStaticPath (videoUUID: string) {
return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.playlistFilename)
private getMasterPlaylistStaticPath (video: MVideo) {
if (isVideoInPrivateDirectory(video.privacy)) {
return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.playlistFilename)
}
return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.playlistFilename)
}
private getSha256SegmentsStaticPath (videoUUID: string) {
return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.segmentsSha256Filename)
private getSha256SegmentsStaticPath (video: MVideo) {
if (isVideoInPrivateDirectory(video.privacy)) {
return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.segmentsSha256Filename)
}
return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.segmentsSha256Filename)
}
}

View File

@ -52,7 +52,7 @@ import {
import { AttributesOnly } from '@shared/typescript-utils'
import { peertubeTruncate } from '../../helpers/core-utils'
import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
import { exists, isBooleanValid } from '../../helpers/custom-validators/misc'
import { exists, isBooleanValid, isUUIDValid } from '../../helpers/custom-validators/misc'
import {
isVideoDescriptionValid,
isVideoDurationValid,
@ -1696,12 +1696,12 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
let files: VideoFile[] = []
if (Array.isArray(this.VideoFiles)) {
const result = videoFilesModelToFormattedJSON(this, this.VideoFiles, includeMagnet)
const result = videoFilesModelToFormattedJSON(this, this.VideoFiles, { includeMagnet })
files = files.concat(result)
}
for (const p of (this.VideoStreamingPlaylists || [])) {
const result = videoFilesModelToFormattedJSON(this, p.VideoFiles, includeMagnet)
const result = videoFilesModelToFormattedJSON(this, p.VideoFiles, { includeMagnet })
files = files.concat(result)
}
@ -1868,22 +1868,14 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
return setAsUpdated('video', this.id, transaction)
}
requiresAuth () {
return this.privacy === VideoPrivacy.PRIVATE || this.privacy === VideoPrivacy.INTERNAL || !!this.VideoBlacklist
}
requiresAuth (paramId: string) {
if (this.privacy === VideoPrivacy.UNLISTED) {
if (!isUUIDValid(paramId)) return true
setPrivacy (newPrivacy: VideoPrivacy) {
if (this.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) {
this.publishedAt = new Date()
return false
}
this.privacy = newPrivacy
}
isConfidential () {
return this.privacy === VideoPrivacy.PRIVATE ||
this.privacy === VideoPrivacy.UNLISTED ||
this.privacy === VideoPrivacy.INTERNAL
return this.privacy === VideoPrivacy.PRIVATE || this.privacy === VideoPrivacy.INTERNAL || !!this.VideoBlacklist
}
async setNewState (newState: VideoState, isNewVideo: boolean, transaction: Transaction) {

View File

@ -34,6 +34,7 @@ import './video-imports'
import './video-playlists'
import './video-source'
import './video-studio'
import './video-token'
import './videos-common-filters'
import './videos-history'
import './videos-overviews'

View File

@ -502,6 +502,23 @@ describe('Test video lives API validator', function () {
await stopFfmpeg(ffmpegCommand)
})
it('Should fail to change live privacy if it has already started', async function () {
this.timeout(40000)
const live = await command.get({ videoId: video.id })
const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey })
await command.waitUntilPublished({ videoId: video.id })
await server.videos.update({
id: video.id,
attributes: { privacy: VideoPrivacy.PUBLIC },
expectedStatus: HttpStatusCode.BAD_REQUEST_400
})
await stopFfmpeg(ffmpegCommand)
})
it('Should fail to stream twice in the save live', async function () {
this.timeout(40000)

View File

@ -1,10 +1,12 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { HttpStatusCode, UserRole } from '@shared/models'
import { getAllFiles } from '@shared/core-utils'
import { HttpStatusCode, UserRole, VideoDetails, VideoPrivacy } from '@shared/models'
import {
cleanupTests,
createMultipleServers,
doubleFollow,
makeRawRequest,
PeerTubeServer,
setAccessTokensToServers,
waitJobs
@ -13,22 +15,9 @@ import {
describe('Test videos files', function () {
let servers: PeerTubeServer[]
let webtorrentId: string
let hlsId: string
let remoteId: string
let userToken: string
let moderatorToken: string
let validId1: string
let validId2: string
let hlsFileId: number
let webtorrentFileId: number
let remoteHLSFileId: number
let remoteWebtorrentFileId: number
// ---------------------------------------------------------------
before(async function () {
@ -41,117 +30,163 @@ describe('Test videos files', function () {
userToken = await servers[0].users.generateUserAndToken('user', UserRole.USER)
moderatorToken = await servers[0].users.generateUserAndToken('moderator', UserRole.MODERATOR)
})
{
const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' })
await waitJobs(servers)
describe('Getting metadata', function () {
let video: VideoDetails
const video = await servers[1].videos.get({ id: uuid })
remoteId = video.uuid
remoteHLSFileId = video.streamingPlaylists[0].files[0].id
remoteWebtorrentFileId = video.files[0].id
}
before(async function () {
const { uuid } = await servers[0].videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
video = await servers[0].videos.getWithToken({ id: uuid })
})
{
await servers[0].config.enableTranscoding(true, true)
it('Should not get metadata of private video without token', async function () {
for (const file of getAllFiles(video)) {
await makeRawRequest({ url: file.metadataUrl, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
}
})
it('Should not get metadata of private video without the appropriate token', async function () {
for (const file of getAllFiles(video)) {
await makeRawRequest({ url: file.metadataUrl, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
}
})
it('Should get metadata of private video with the appropriate token', async function () {
for (const file of getAllFiles(video)) {
await makeRawRequest({ url: file.metadataUrl, token: servers[0].accessToken, expectedStatus: HttpStatusCode.OK_200 })
}
})
})
describe('Deleting files', function () {
let webtorrentId: string
let hlsId: string
let remoteId: string
let validId1: string
let validId2: string
let hlsFileId: number
let webtorrentFileId: number
let remoteHLSFileId: number
let remoteWebtorrentFileId: number
before(async function () {
this.timeout(300_000)
{
const { uuid } = await servers[0].videos.quickUpload({ name: 'both 1' })
const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' })
await waitJobs(servers)
const video = await servers[0].videos.get({ id: uuid })
validId1 = video.uuid
hlsFileId = video.streamingPlaylists[0].files[0].id
webtorrentFileId = video.files[0].id
const video = await servers[1].videos.get({ id: uuid })
remoteId = video.uuid
remoteHLSFileId = video.streamingPlaylists[0].files[0].id
remoteWebtorrentFileId = video.files[0].id
}
{
const { uuid } = await servers[0].videos.quickUpload({ name: 'both 2' })
validId2 = uuid
await servers[0].config.enableTranscoding(true, true)
{
const { uuid } = await servers[0].videos.quickUpload({ name: 'both 1' })
await waitJobs(servers)
const video = await servers[0].videos.get({ id: uuid })
validId1 = video.uuid
hlsFileId = video.streamingPlaylists[0].files[0].id
webtorrentFileId = video.files[0].id
}
{
const { uuid } = await servers[0].videos.quickUpload({ name: 'both 2' })
validId2 = uuid
}
}
}
await waitJobs(servers)
await waitJobs(servers)
{
await servers[0].config.enableTranscoding(false, true)
const { uuid } = await servers[0].videos.quickUpload({ name: 'hls' })
hlsId = uuid
}
{
await servers[0].config.enableTranscoding(false, true)
const { uuid } = await servers[0].videos.quickUpload({ name: 'hls' })
hlsId = uuid
}
await waitJobs(servers)
await waitJobs(servers)
{
await servers[0].config.enableTranscoding(false, true)
const { uuid } = await servers[0].videos.quickUpload({ name: 'webtorrent' })
webtorrentId = uuid
}
{
await servers[0].config.enableTranscoding(false, true)
const { uuid } = await servers[0].videos.quickUpload({ name: 'webtorrent' })
webtorrentId = uuid
}
await waitJobs(servers)
})
await waitJobs(servers)
})
it('Should not delete files of a unknown video', async function () {
const expectedStatus = HttpStatusCode.NOT_FOUND_404
it('Should not delete files of a unknown video', async function () {
const expectedStatus = HttpStatusCode.NOT_FOUND_404
await servers[0].videos.removeHLSPlaylist({ videoId: 404, expectedStatus })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: 404, expectedStatus })
await servers[0].videos.removeHLSPlaylist({ videoId: 404, expectedStatus })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: 404, expectedStatus })
await servers[0].videos.removeHLSFile({ videoId: 404, fileId: hlsFileId, expectedStatus })
await servers[0].videos.removeWebTorrentFile({ videoId: 404, fileId: webtorrentFileId, expectedStatus })
})
await servers[0].videos.removeHLSFile({ videoId: 404, fileId: hlsFileId, expectedStatus })
await servers[0].videos.removeWebTorrentFile({ videoId: 404, fileId: webtorrentFileId, expectedStatus })
})
it('Should not delete unknown files', async function () {
const expectedStatus = HttpStatusCode.NOT_FOUND_404
it('Should not delete unknown files', async function () {
const expectedStatus = HttpStatusCode.NOT_FOUND_404
await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: webtorrentFileId, expectedStatus })
await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: hlsFileId, expectedStatus })
})
await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: webtorrentFileId, expectedStatus })
await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: hlsFileId, expectedStatus })
})
it('Should not delete files of a remote video', async function () {
const expectedStatus = HttpStatusCode.BAD_REQUEST_400
it('Should not delete files of a remote video', async function () {
const expectedStatus = HttpStatusCode.BAD_REQUEST_400
await servers[0].videos.removeHLSPlaylist({ videoId: remoteId, expectedStatus })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: remoteId, expectedStatus })
await servers[0].videos.removeHLSPlaylist({ videoId: remoteId, expectedStatus })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: remoteId, expectedStatus })
await servers[0].videos.removeHLSFile({ videoId: remoteId, fileId: remoteHLSFileId, expectedStatus })
await servers[0].videos.removeWebTorrentFile({ videoId: remoteId, fileId: remoteWebtorrentFileId, expectedStatus })
})
await servers[0].videos.removeHLSFile({ videoId: remoteId, fileId: remoteHLSFileId, expectedStatus })
await servers[0].videos.removeWebTorrentFile({ videoId: remoteId, fileId: remoteWebtorrentFileId, expectedStatus })
})
it('Should not delete files by a non admin user', async function () {
const expectedStatus = HttpStatusCode.FORBIDDEN_403
it('Should not delete files by a non admin user', async function () {
const expectedStatus = HttpStatusCode.FORBIDDEN_403
await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: userToken, expectedStatus })
await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: moderatorToken, expectedStatus })
await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: userToken, expectedStatus })
await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: moderatorToken, expectedStatus })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1, token: userToken, expectedStatus })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1, token: moderatorToken, expectedStatus })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1, token: userToken, expectedStatus })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1, token: moderatorToken, expectedStatus })
await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: userToken, expectedStatus })
await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: moderatorToken, expectedStatus })
await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: userToken, expectedStatus })
await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: moderatorToken, expectedStatus })
await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId, token: userToken, expectedStatus })
await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId, token: moderatorToken, expectedStatus })
})
await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId, token: userToken, expectedStatus })
await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId, token: moderatorToken, expectedStatus })
})
it('Should not delete files if the files are not available', async function () {
await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
it('Should not delete files if the files are not available', async function () {
await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
await servers[0].videos.removeWebTorrentFile({ videoId: webtorrentId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
})
await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
await servers[0].videos.removeWebTorrentFile({ videoId: webtorrentId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
})
it('Should not delete files if no both versions are available', async function () {
await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
})
it('Should not delete files if no both versions are available', async function () {
await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
})
it('Should delete files if both versions are available', async function () {
await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId })
await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId })
it('Should delete files if both versions are available', async function () {
await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId })
await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId })
await servers[0].videos.removeHLSPlaylist({ videoId: validId1 })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId2 })
await servers[0].videos.removeHLSPlaylist({ videoId: validId1 })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId2 })
})
})
after(async function () {

View File

@ -0,0 +1,44 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { HttpStatusCode, VideoPrivacy } from '@shared/models'
import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
describe('Test video tokens', function () {
let server: PeerTubeServer
let videoId: string
let userToken: string
// ---------------------------------------------------------------
before(async function () {
this.timeout(300_000)
server = await createSingleServer(1)
await setAccessTokensToServers([ server ])
const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
videoId = uuid
userToken = await server.users.generateUserAndToken('user1')
})
it('Should not generate tokens for unauthenticated user', async function () {
await server.videoToken.create({ videoId, token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
})
it('Should not generate tokens of unknown video', async function () {
await server.videoToken.create({ videoId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
})
it('Should not generate tokens of a non owned video', async function () {
await server.videoToken.create({ videoId, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
})
it('Should generate token', async function () {
await server.videoToken.create({ videoId })
})
after(async function () {
await cleanupTests([ server ])
})
})

View File

@ -79,8 +79,8 @@ describe('Fast restream in live', function () {
expect(video.streamingPlaylists).to.have.lengthOf(1)
await server.live.getSegmentFile({ videoUUID: liveId, segment: 0, playlistNumber: 0 })
await makeRawRequest(video.streamingPlaylists[0].playlistUrl, HttpStatusCode.OK_200)
await makeRawRequest(video.streamingPlaylists[0].segmentsSha256Url, HttpStatusCode.OK_200)
await makeRawRequest({ url: video.streamingPlaylists[0].playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: video.streamingPlaylists[0].segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
await wait(100)
}

View File

@ -21,6 +21,7 @@ import {
doubleFollow,
killallServers,
LiveCommand,
makeGetRequest,
makeRawRequest,
PeerTubeServer,
sendRTMPStream,
@ -157,8 +158,8 @@ describe('Test live', function () {
expect(video.privacy.id).to.equal(VideoPrivacy.UNLISTED)
expect(video.nsfw).to.be.true
await makeRawRequest(server.url + video.thumbnailPath, HttpStatusCode.OK_200)
await makeRawRequest(server.url + video.previewPath, HttpStatusCode.OK_200)
await makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
await makeGetRequest({ url: server.url, path: video.previewPath, expectedStatus: HttpStatusCode.OK_200 })
}
})
@ -532,8 +533,8 @@ describe('Test live', function () {
expect(video.files).to.have.lengthOf(0)
const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS)
await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200)
await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200)
await makeRawRequest({ url: hlsPlaylist.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: hlsPlaylist.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
// We should have generated random filenames
expect(basename(hlsPlaylist.playlistUrl)).to.not.equal('master.m3u8')
@ -564,8 +565,8 @@ describe('Test live', function () {
expect(probe.format.bit_rate).to.be.below(maxBitrateLimits[videoStream.height])
expect(probe.format.bit_rate).to.be.at.least(minBitrateLimits[videoStream.height])
await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200)
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: file.torrentUrl, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
}
})

View File

@ -48,7 +48,7 @@ async function checkFilesExist (servers: PeerTubeServer[], videoUUID: string, nu
for (const file of files) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
}
}

View File

@ -66,7 +66,7 @@ describe('Object storage for video import', function () {
const fileUrl = video.files[0].fileUrl
expectStartWith(fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
await makeRawRequest(fileUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.OK_200 })
})
})
@ -91,13 +91,13 @@ describe('Object storage for video import', function () {
for (const file of video.files) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
for (const file of video.streamingPlaylists[0].files) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
})
})

View File

@ -59,11 +59,11 @@ async function checkFiles (options: {
expectStartWith(file.fileUrl, start)
const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
const location = res.headers['location']
expectStartWith(location, start)
await makeRawRequest(location, HttpStatusCode.OK_200)
await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
}
const hls = video.streamingPlaylists[0]
@ -81,19 +81,19 @@ async function checkFiles (options: {
expectStartWith(hls.playlistUrl, start)
expectStartWith(hls.segmentsSha256Url, start)
await makeRawRequest(hls.playlistUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
const resSha = await makeRawRequest(hls.segmentsSha256Url, HttpStatusCode.OK_200)
const resSha = await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
expect(JSON.stringify(resSha.body)).to.not.throw
for (const file of hls.files) {
expectStartWith(file.fileUrl, start)
const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
const location = res.headers['location']
expectStartWith(location, start)
await makeRawRequest(location, HttpStatusCode.OK_200)
await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
}
}
@ -104,7 +104,7 @@ async function checkFiles (options: {
expect(torrent.files.length).to.equal(1)
expect(torrent.files[0].path).to.exist.and.to.not.equal('')
const res = await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
const res = await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
expect(res.body).to.have.length.above(100)
}
@ -220,7 +220,7 @@ function runTestSuite (options: {
it('Should fetch correctly all the files', async function () {
for (const url of deletedUrls.concat(keptUrls)) {
await makeRawRequest(url, HttpStatusCode.OK_200)
await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
}
})
@ -231,13 +231,13 @@ function runTestSuite (options: {
await waitJobs(servers)
for (const url of deletedUrls) {
await makeRawRequest(url, HttpStatusCode.NOT_FOUND_404)
await makeRawRequest({ url, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
}
})
it('Should have kept other files', async function () {
for (const url of keptUrls) {
await makeRawRequest(url, HttpStatusCode.OK_200)
await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
}
})

View File

@ -39,7 +39,7 @@ async function checkMagnetWebseeds (file: VideoFile, baseWebseeds: string[], ser
expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
for (const url of parsed.urlList) {
await makeRawRequest(url, HttpStatusCode.OK_200)
await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
}
}

View File

@ -18,7 +18,7 @@ describe('Open Telemetry', function () {
let hasError = false
try {
await makeRawRequest(metricsUrl, HttpStatusCode.NOT_FOUND_404)
await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
} catch (err) {
hasError = err.message.includes('ECONNREFUSED')
}
@ -37,7 +37,7 @@ describe('Open Telemetry', function () {
}
})
const res = await makeRawRequest(metricsUrl, HttpStatusCode.OK_200)
const res = await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.OK_200 })
expect(res.text).to.contain('peertube_job_queue_total{')
})
@ -60,7 +60,7 @@ describe('Open Telemetry', function () {
}
})
const res = await makeRawRequest(metricsUrl, HttpStatusCode.OK_200)
const res = await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.OK_200 })
expect(res.text).to.contain('peertube_playback_http_downloaded_bytes_total{')
})

View File

@ -20,7 +20,7 @@ import {
async function checkFilesInObjectStorage (video: VideoDetails) {
for (const file of video.files) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
if (video.streamingPlaylists.length === 0) return
@ -28,14 +28,14 @@ async function checkFilesInObjectStorage (video: VideoDetails) {
const hlsPlaylist = video.streamingPlaylists[0]
for (const file of hlsPlaylist.files) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
expectStartWith(hlsPlaylist.playlistUrl, ObjectStorageCommand.getPlaylistBaseUrl())
await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: hlsPlaylist.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
expectStartWith(hlsPlaylist.segmentsSha256Url, ObjectStorageCommand.getPlaylistBaseUrl())
await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200)
await makeRawRequest({ url: hlsPlaylist.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
}
function runTests (objectStorage: boolean) {
@ -234,7 +234,7 @@ function runTests (objectStorage: boolean) {
it('Should have correctly deleted previous files', async function () {
for (const fileUrl of shouldBeDeleted) {
await makeRawRequest(fileUrl, HttpStatusCode.NOT_FOUND_404)
await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
}
})

View File

@ -1,168 +1,48 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { expect } from 'chai'
import { basename, join } from 'path'
import {
checkDirectoryIsEmpty,
checkResolutionsInMasterPlaylist,
checkSegmentHash,
checkTmpIsEmpty,
expectStartWith,
hlsInfohashExist
} from '@server/tests/shared'
import { areObjectStorageTestsDisabled, removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils'
import { HttpStatusCode, VideoStreamingPlaylistType } from '@shared/models'
import { join } from 'path'
import { checkDirectoryIsEmpty, checkTmpIsEmpty, completeCheckHlsPlaylist } from '@server/tests/shared'
import { areObjectStorageTestsDisabled } from '@shared/core-utils'
import { HttpStatusCode } from '@shared/models'
import {
cleanupTests,
createMultipleServers,
doubleFollow,
makeRawRequest,
ObjectStorageCommand,
PeerTubeServer,
setAccessTokensToServers,
waitJobs,
webtorrentAdd
waitJobs
} from '@shared/server-commands'
import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants'
async function checkHlsPlaylist (options: {
servers: PeerTubeServer[]
videoUUID: string
hlsOnly: boolean
resolutions?: number[]
objectStorageBaseUrl: string
}) {
const { videoUUID, hlsOnly, objectStorageBaseUrl } = options
const resolutions = options.resolutions ?? [ 240, 360, 480, 720 ]
for (const server of options.servers) {
const videoDetails = await server.videos.get({ id: videoUUID })
const baseUrl = `http://${videoDetails.account.host}`
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
const hlsPlaylist = videoDetails.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
expect(hlsPlaylist).to.not.be.undefined
const hlsFiles = hlsPlaylist.files
expect(hlsFiles).to.have.lengthOf(resolutions.length)
if (hlsOnly) expect(videoDetails.files).to.have.lengthOf(0)
else expect(videoDetails.files).to.have.lengthOf(resolutions.length)
// Check JSON files
for (const resolution of resolutions) {
const file = hlsFiles.find(f => f.resolution.id === resolution)
expect(file).to.not.be.undefined
expect(file.magnetUri).to.have.lengthOf.above(2)
expect(file.torrentUrl).to.match(
new RegExp(`http://${server.host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}-hls.torrent`)
)
if (objectStorageBaseUrl) {
expectStartWith(file.fileUrl, objectStorageBaseUrl)
} else {
expect(file.fileUrl).to.match(
new RegExp(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${uuidRegex}-${file.resolution.id}-fragmented.mp4`)
)
}
expect(file.resolution.label).to.equal(resolution + 'p')
await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200)
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
const torrent = await webtorrentAdd(file.magnetUri, true)
expect(torrent.files).to.be.an('array')
expect(torrent.files.length).to.equal(1)
expect(torrent.files[0].path).to.exist.and.to.not.equal('')
}
// Check master playlist
{
await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions })
const masterPlaylist = await server.streamingPlaylists.get({ url: hlsPlaylist.playlistUrl })
let i = 0
for (const resolution of resolutions) {
expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
const url = 'http://' + videoDetails.account.host
await hlsInfohashExist(url, hlsPlaylist.playlistUrl, i)
i++
}
}
// Check resolution playlists
{
for (const resolution of resolutions) {
const file = hlsFiles.find(f => f.resolution.id === resolution)
const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8'
const url = objectStorageBaseUrl
? `${objectStorageBaseUrl}hls/${videoUUID}/${playlistName}`
: `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}`
const subPlaylist = await server.streamingPlaylists.get({ url })
expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`))
expect(subPlaylist).to.contain(basename(file.fileUrl))
}
}
{
const baseUrlAndPath = objectStorageBaseUrl
? objectStorageBaseUrl + 'hls/' + videoUUID
: baseUrl + '/static/streaming-playlists/hls/' + videoUUID
for (const resolution of resolutions) {
await checkSegmentHash({
server,
baseUrlPlaylist: baseUrlAndPath,
baseUrlSegment: baseUrlAndPath,
resolution,
hlsPlaylist
})
}
}
}
}
describe('Test HLS videos', function () {
let servers: PeerTubeServer[] = []
let videoUUID = ''
let videoAudioUUID = ''
function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) {
const videoUUIDs: string[] = []
it('Should upload a video and transcode it to HLS', async function () {
this.timeout(120000)
const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video 1', fixture: 'video_short.webm' } })
videoUUID = uuid
videoUUIDs.push(uuid)
await waitJobs(servers)
await checkHlsPlaylist({ servers, videoUUID, hlsOnly, objectStorageBaseUrl })
await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl })
})
it('Should upload an audio file and transcode it to HLS', async function () {
this.timeout(120000)
const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video audio', fixture: 'sample.ogg' } })
videoAudioUUID = uuid
videoUUIDs.push(uuid)
await waitJobs(servers)
await checkHlsPlaylist({
await completeCheckHlsPlaylist({
servers,
videoUUID: videoAudioUUID,
videoUUID: uuid,
hlsOnly,
resolutions: [ DEFAULT_AUDIO_RESOLUTION, 360, 240 ],
objectStorageBaseUrl
@ -172,31 +52,36 @@ describe('Test HLS videos', function () {
it('Should update the video', async function () {
this.timeout(30000)
await servers[0].videos.update({ id: videoUUID, attributes: { name: 'video 1 updated' } })
await servers[0].videos.update({ id: videoUUIDs[0], attributes: { name: 'video 1 updated' } })
await waitJobs(servers)
await checkHlsPlaylist({ servers, videoUUID, hlsOnly, objectStorageBaseUrl })
await completeCheckHlsPlaylist({ servers, videoUUID: videoUUIDs[0], hlsOnly, objectStorageBaseUrl })
})
it('Should delete videos', async function () {
this.timeout(10000)
await servers[0].videos.remove({ id: videoUUID })
await servers[0].videos.remove({ id: videoAudioUUID })
for (const uuid of videoUUIDs) {
await servers[0].videos.remove({ id: uuid })
}
await waitJobs(servers)
for (const server of servers) {
await server.videos.get({ id: videoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
await server.videos.get({ id: videoAudioUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
for (const uuid of videoUUIDs) {
await server.videos.get({ id: uuid, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
}
}
})
it('Should have the playlists/segment deleted from the disk', async function () {
for (const server of servers) {
await checkDirectoryIsEmpty(server, 'videos')
await checkDirectoryIsEmpty(server, join('streaming-playlists', 'hls'))
await checkDirectoryIsEmpty(server, 'videos', [ 'private' ])
await checkDirectoryIsEmpty(server, join('videos', 'private'))
await checkDirectoryIsEmpty(server, join('streaming-playlists', 'hls'), [ 'private' ])
await checkDirectoryIsEmpty(server, join('streaming-playlists', 'hls', 'private'))
}
})

View File

@ -2,4 +2,5 @@ export * from './audio-only'
export * from './create-transcoding'
export * from './hls'
export * from './transcoder'
export * from './update-while-transcoding'
export * from './video-studio'

View File

@ -0,0 +1,151 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { completeCheckHlsPlaylist } from '@server/tests/shared'
import { areObjectStorageTestsDisabled, wait } from '@shared/core-utils'
import { VideoPrivacy } from '@shared/models'
import {
cleanupTests,
createMultipleServers,
doubleFollow,
ObjectStorageCommand,
PeerTubeServer,
setAccessTokensToServers,
waitJobs
} from '@shared/server-commands'
describe('Test update video privacy while transcoding', function () {
let servers: PeerTubeServer[] = []
const videoUUIDs: string[] = []
function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) {
it('Should not have an error while quickly updating a private video to public after upload #1', async function () {
this.timeout(360_000)
const attributes = {
name: 'quick update',
privacy: VideoPrivacy.PRIVATE
}
const { uuid } = await servers[0].videos.upload({ attributes, waitTorrentGeneration: false })
await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } })
videoUUIDs.push(uuid)
await waitJobs(servers)
await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl })
})
it('Should not have an error while quickly updating a private video to public after upload #2', async function () {
{
const attributes = {
name: 'quick update 2',
privacy: VideoPrivacy.PRIVATE
}
const { uuid } = await servers[0].videos.upload({ attributes, waitTorrentGeneration: true })
await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } })
videoUUIDs.push(uuid)
await waitJobs(servers)
await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl })
}
})
it('Should not have an error while quickly updating a private video to public after upload #3', async function () {
const attributes = {
name: 'quick update 3',
privacy: VideoPrivacy.PRIVATE
}
const { uuid } = await servers[0].videos.upload({ attributes, waitTorrentGeneration: true })
await wait(1000)
await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } })
videoUUIDs.push(uuid)
await waitJobs(servers)
await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl })
})
}
before(async function () {
this.timeout(120000)
const configOverride = {
transcoding: {
enabled: true,
allow_audio_files: true,
hls: {
enabled: true
}
}
}
servers = await createMultipleServers(2, configOverride)
// Get the access tokens
await setAccessTokensToServers(servers)
// Server 1 and server 2 follow each other
await doubleFollow(servers[0], servers[1])
})
describe('With WebTorrent & HLS enabled', function () {
runTestSuite(false)
})
describe('With only HLS enabled', function () {
before(async function () {
await servers[0].config.updateCustomSubConfig({
newConfig: {
transcoding: {
enabled: true,
allowAudioFiles: true,
resolutions: {
'144p': false,
'240p': true,
'360p': true,
'480p': true,
'720p': true,
'1080p': true,
'1440p': true,
'2160p': true
},
hls: {
enabled: true
},
webtorrent: {
enabled: false
}
}
}
})
})
runTestSuite(true)
})
describe('With object storage enabled', function () {
if (areObjectStorageTestsDisabled()) return
before(async function () {
this.timeout(120000)
const configOverride = ObjectStorageCommand.getDefaultConfig()
await ObjectStorageCommand.prepareDefaultBuckets()
await servers[0].kill()
await servers[0].run(configOverride)
})
runTestSuite(true, ObjectStorageCommand.getPlaylistBaseUrl())
})
after(async function () {
await cleanupTests(servers)
})
})

View File

@ -19,3 +19,4 @@ import './videos-common-filters'
import './videos-history'
import './videos-overview'
import './video-source'
import './video-static-file-privacy'

View File

@ -153,7 +153,7 @@ describe('Test videos files', function () {
expect(video.streamingPlaylists[0].files).to.have.lengthOf(files.length - 1)
expect(video.streamingPlaylists[0].files.find(f => f.id === toDelete.id)).to.not.exist
const { text } = await makeRawRequest(video.streamingPlaylists[0].playlistUrl)
const { text } = await makeRawRequest({ url: video.streamingPlaylists[0].playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
expect(text.includes(`-${toDelete.resolution.id}.m3u8`)).to.be.false
expect(text.includes(`-${video.streamingPlaylists[0].files[0].resolution.id}.m3u8`)).to.be.true

View File

@ -0,0 +1,389 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { expect } from 'chai'
import { decode } from 'magnet-uri'
import { expectStartWith } from '@server/tests/shared'
import { getAllFiles, wait } from '@shared/core-utils'
import { HttpStatusCode, LiveVideo, VideoDetails, VideoPrivacy } from '@shared/models'
import {
cleanupTests,
createSingleServer,
findExternalSavedVideo,
makeRawRequest,
parseTorrentVideo,
PeerTubeServer,
sendRTMPStream,
setAccessTokensToServers,
setDefaultVideoChannel,
stopFfmpeg,
waitJobs
} from '@shared/server-commands'
describe('Test video static file privacy', function () {
let server: PeerTubeServer
let userToken: string
before(async function () {
this.timeout(50000)
server = await createSingleServer(1)
await setAccessTokensToServers([ server ])
await setDefaultVideoChannel([ server ])
userToken = await server.users.generateUserAndToken('user1')
})
describe('VOD static file path', function () {
function runSuite () {
async function checkPrivateWebTorrentFiles (uuid: string) {
const video = await server.videos.getWithToken({ id: uuid })
for (const file of video.files) {
expect(file.fileDownloadUrl).to.not.include('/private/')
expectStartWith(file.fileUrl, server.url + '/static/webseed/private/')
const torrent = await parseTorrentVideo(server, file)
expect(torrent.urlList).to.have.lengthOf(0)
const magnet = decode(file.magnetUri)
expect(magnet.urlList).to.have.lengthOf(0)
await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
}
const hls = video.streamingPlaylists[0]
if (hls) {
expectStartWith(hls.playlistUrl, server.url + '/static/streaming-playlists/hls/private/')
expectStartWith(hls.segmentsSha256Url, server.url + '/static/streaming-playlists/hls/private/')
await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
}
}
async function checkPublicWebTorrentFiles (uuid: string) {
const video = await server.videos.get({ id: uuid })
for (const file of getAllFiles(video)) {
expect(file.fileDownloadUrl).to.not.include('/private/')
expect(file.fileUrl).to.not.include('/private/')
const torrent = await parseTorrentVideo(server, file)
expect(torrent.urlList[0]).to.not.include('private')
const magnet = decode(file.magnetUri)
expect(magnet.urlList[0]).to.not.include('private')
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: torrent.urlList[0], expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: magnet.urlList[0], expectedStatus: HttpStatusCode.OK_200 })
}
const hls = video.streamingPlaylists[0]
if (hls) {
expect(hls.playlistUrl).to.not.include('private')
expect(hls.segmentsSha256Url).to.not.include('private')
await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
}
}
it('Should upload a private/internal video and have a private static path', async function () {
this.timeout(120000)
for (const privacy of [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]) {
const { uuid } = await server.videos.quickUpload({ name: 'video', privacy })
await waitJobs([ server ])
await checkPrivateWebTorrentFiles(uuid)
}
})
it('Should upload a public video and update it as private/internal to have a private static path', async function () {
this.timeout(120000)
for (const privacy of [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]) {
const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PUBLIC })
await waitJobs([ server ])
await server.videos.update({ id: uuid, attributes: { privacy } })
await waitJobs([ server ])
await checkPrivateWebTorrentFiles(uuid)
}
})
it('Should upload a private video and update it to unlisted to have a public static path', async function () {
this.timeout(120000)
const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
await waitJobs([ server ])
await server.videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.UNLISTED } })
await waitJobs([ server ])
await checkPublicWebTorrentFiles(uuid)
})
it('Should upload an internal video and update it to public to have a public static path', async function () {
this.timeout(120000)
const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.INTERNAL })
await waitJobs([ server ])
await server.videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } })
await waitJobs([ server ])
await checkPublicWebTorrentFiles(uuid)
})
it('Should upload an internal video and schedule a public publish', async function () {
this.timeout(120000)
const attributes = {
name: 'video',
privacy: VideoPrivacy.PRIVATE,
scheduleUpdate: {
updateAt: new Date(Date.now() + 1000).toISOString(),
privacy: VideoPrivacy.PUBLIC as VideoPrivacy.PUBLIC
}
}
const { uuid } = await server.videos.upload({ attributes })
await waitJobs([ server ])
await wait(1000)
await server.debug.sendCommand({ body: { command: 'process-update-videos-scheduler' } })
await waitJobs([ server ])
await checkPublicWebTorrentFiles(uuid)
})
}
describe('Without transcoding', function () {
runSuite()
})
describe('With transcoding', function () {
before(async function () {
await server.config.enableMinimumTranscoding()
})
runSuite()
})
})
describe('VOD static file right check', function () {
let unrelatedFileToken: string
async function checkVideoFiles (options: {
id: string
expectedStatus: HttpStatusCode
token: string
videoFileToken: string
}) {
const { id, expectedStatus, token, videoFileToken } = options
const video = await server.videos.getWithToken({ id })
for (const file of getAllFiles(video)) {
await makeRawRequest({ url: file.fileUrl, token, expectedStatus })
await makeRawRequest({ url: file.fileDownloadUrl, token, expectedStatus })
await makeRawRequest({ url: file.fileUrl, query: { videoFileToken }, expectedStatus })
await makeRawRequest({ url: file.fileDownloadUrl, query: { videoFileToken }, expectedStatus })
}
const hls = video.streamingPlaylists[0]
await makeRawRequest({ url: hls.playlistUrl, token, expectedStatus })
await makeRawRequest({ url: hls.segmentsSha256Url, token, expectedStatus })
await makeRawRequest({ url: hls.playlistUrl, query: { videoFileToken }, expectedStatus })
await makeRawRequest({ url: hls.segmentsSha256Url, query: { videoFileToken }, expectedStatus })
}
before(async function () {
await server.config.enableMinimumTranscoding()
const { uuid } = await server.videos.quickUpload({ name: 'another video' })
unrelatedFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid })
})
it('Should not be able to access a private video files without OAuth token and file token', async function () {
this.timeout(120000)
const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.INTERNAL })
await waitJobs([ server ])
await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.FORBIDDEN_403, token: null, videoFileToken: null })
})
it('Should not be able to access an internal video files without appropriate OAuth token and file token', async function () {
this.timeout(120000)
const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
await waitJobs([ server ])
await checkVideoFiles({
id: uuid,
expectedStatus: HttpStatusCode.FORBIDDEN_403,
token: userToken,
videoFileToken: unrelatedFileToken
})
})
it('Should be able to access a private video files with appropriate OAuth token or file token', async function () {
this.timeout(120000)
const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid })
await waitJobs([ server ])
await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken, videoFileToken })
})
it('Should be able to access a private video of another user with an admin OAuth token or file token', async function () {
this.timeout(120000)
const { uuid } = await server.videos.quickUpload({ name: 'video', token: userToken, privacy: VideoPrivacy.PRIVATE })
const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid })
await waitJobs([ server ])
await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken, videoFileToken })
})
})
describe('Live static file path and check', function () {
let normalLiveId: string
let normalLive: LiveVideo
let permanentLiveId: string
let permanentLive: LiveVideo
let unrelatedFileToken: string
async function checkLiveFiles (live: LiveVideo, liveId: string) {
const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey })
await server.live.waitUntilPublished({ videoId: liveId })
const video = await server.videos.getWithToken({ id: liveId })
const fileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid })
const hls = video.streamingPlaylists[0]
for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) {
expectStartWith(url, server.url + '/static/streaming-playlists/hls/private/')
await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
}
await stopFfmpeg(ffmpegCommand)
}
async function checkReplay (replay: VideoDetails) {
const fileToken = await server.videoToken.getVideoFileToken({ videoId: replay.uuid })
const hls = replay.streamingPlaylists[0]
expect(hls.files).to.not.have.lengthOf(0)
for (const file of hls.files) {
await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: file.fileUrl, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: file.fileUrl, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await makeRawRequest({
url: file.fileUrl,
query: { videoFileToken: unrelatedFileToken },
expectedStatus: HttpStatusCode.FORBIDDEN_403
})
}
for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) {
expectStartWith(url, server.url + '/static/streaming-playlists/hls/private/')
await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
}
}
before(async function () {
await server.config.enableMinimumTranscoding()
const { uuid } = await server.videos.quickUpload({ name: 'another video' })
unrelatedFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid })
await server.config.enableLive({
allowReplay: true,
transcoding: true,
resolutions: 'min'
})
{
const { video, live } = await server.live.quickCreate({ saveReplay: true, permanentLive: false, privacy: VideoPrivacy.PRIVATE })
normalLiveId = video.uuid
normalLive = live
}
{
const { video, live } = await server.live.quickCreate({ saveReplay: true, permanentLive: true, privacy: VideoPrivacy.PRIVATE })
permanentLiveId = video.uuid
permanentLive = live
}
})
it('Should create a private normal live and have a private static path', async function () {
this.timeout(240000)
await checkLiveFiles(normalLive, normalLiveId)
})
it('Should create a private permanent live and have a private static path', async function () {
this.timeout(240000)
await checkLiveFiles(permanentLive, permanentLiveId)
})
it('Should have created a replay of the normal live with a private static path', async function () {
this.timeout(240000)
await server.live.waitUntilReplacedByReplay({ videoId: normalLiveId })
const replay = await server.videos.getWithToken({ id: normalLiveId })
await checkReplay(replay)
})
it('Should have created a replay of the permanent live with a private static path', async function () {
this.timeout(240000)
await server.live.waitUntilWaiting({ videoId: permanentLiveId })
await waitJobs([ server ])
const live = await server.videos.getWithToken({ id: permanentLiveId })
const replayFromList = await findExternalSavedVideo(server, live)
const replay = await server.videos.getWithToken({ id: replayFromList.id })
await checkReplay(replay)
})
})
after(async function () {
await cleanupTests([ server ])
})
})

View File

@ -29,7 +29,7 @@ async function checkFiles (video: VideoDetails, objectStorage: boolean) {
for (const file of video.files) {
if (objectStorage) expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
}

View File

@ -22,7 +22,7 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObject
expectStartWith(file.fileUrl, start)
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
const start = inObjectStorage
@ -36,7 +36,7 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObject
for (const file of hls.files) {
expectStartWith(file.fileUrl, start)
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
}

View File

@ -23,7 +23,7 @@ async function checkFilesInObjectStorage (files: VideoFile[], type: 'webtorrent'
expectStartWith(file.fileUrl, shouldStartWith)
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
}

View File

@ -5,7 +5,7 @@ import { createFile, readdir } from 'fs-extra'
import { join } from 'path'
import { wait } from '@shared/core-utils'
import { buildUUID } from '@shared/extra-utils'
import { HttpStatusCode, VideoPlaylistPrivacy } from '@shared/models'
import { HttpStatusCode, VideoPlaylistPrivacy, VideoPrivacy } from '@shared/models'
import {
cleanupTests,
CLICommand,
@ -36,22 +36,28 @@ async function assertNotExists (server: PeerTubeServer, directory: string, subst
async function assertCountAreOkay (servers: PeerTubeServer[]) {
for (const server of servers) {
const videosCount = await countFiles(server, 'videos')
expect(videosCount).to.equal(8)
expect(videosCount).to.equal(9) // 2 videos with 4 resolutions + private directory
const privateVideosCount = await countFiles(server, 'videos/private')
expect(privateVideosCount).to.equal(4)
const torrentsCount = await countFiles(server, 'torrents')
expect(torrentsCount).to.equal(16)
expect(torrentsCount).to.equal(24)
const previewsCount = await countFiles(server, 'previews')
expect(previewsCount).to.equal(2)
expect(previewsCount).to.equal(3)
const thumbnailsCount = await countFiles(server, 'thumbnails')
expect(thumbnailsCount).to.equal(6)
expect(thumbnailsCount).to.equal(7) // 3 local videos, 1 local playlist, 2 remotes videos and 1 remote playlist
const avatarsCount = await countFiles(server, 'avatars')
expect(avatarsCount).to.equal(4)
const hlsRootCount = await countFiles(server, 'streaming-playlists/hls')
expect(hlsRootCount).to.equal(2)
const hlsRootCount = await countFiles(server, join('streaming-playlists', 'hls'))
expect(hlsRootCount).to.equal(3) // 2 videos + private directory
const hlsPrivateRootCount = await countFiles(server, join('streaming-playlists', 'hls', 'private'))
expect(hlsPrivateRootCount).to.equal(1)
}
}
@ -67,8 +73,10 @@ describe('Test prune storage scripts', function () {
await setDefaultVideoChannel(servers)
for (const server of servers) {
await server.videos.upload({ attributes: { name: 'video 1' } })
await server.videos.upload({ attributes: { name: 'video 2' } })
await server.videos.upload({ attributes: { name: 'video 1', privacy: VideoPrivacy.PUBLIC } })
await server.videos.upload({ attributes: { name: 'video 2', privacy: VideoPrivacy.PUBLIC } })
await server.videos.upload({ attributes: { name: 'video 3', privacy: VideoPrivacy.PRIVATE } })
await server.users.updateMyAvatar({ fixture: 'avatar.png' })
@ -123,13 +131,16 @@ describe('Test prune storage scripts', function () {
it('Should create some dirty files', async function () {
for (let i = 0; i < 2; i++) {
{
const base = servers[0].servers.buildDirectory('videos')
const basePublic = servers[0].servers.buildDirectory('videos')
const basePrivate = servers[0].servers.buildDirectory(join('videos', 'private'))
const n1 = buildUUID() + '.mp4'
const n2 = buildUUID() + '.webm'
await createFile(join(base, n1))
await createFile(join(base, n2))
await createFile(join(basePublic, n1))
await createFile(join(basePublic, n2))
await createFile(join(basePrivate, n1))
await createFile(join(basePrivate, n2))
badNames['videos'] = [ n1, n2 ]
}
@ -184,10 +195,12 @@ describe('Test prune storage scripts', function () {
{
const directory = join('streaming-playlists', 'hls')
const base = servers[0].servers.buildDirectory(directory)
const basePublic = servers[0].servers.buildDirectory(directory)
const basePrivate = servers[0].servers.buildDirectory(join(directory, 'private'))
const n1 = buildUUID()
await createFile(join(base, n1))
await createFile(join(basePublic, n1))
await createFile(join(basePrivate, n1))
badNames[directory] = [ n1 ]
}
}

View File

@ -6,7 +6,7 @@ import {
cleanupTests,
createMultipleServers,
doubleFollow,
makeRawRequest,
makeGetRequest,
PeerTubeServer,
setAccessTokensToServers,
waitJobs
@ -16,8 +16,8 @@ async function testThumbnail (server: PeerTubeServer, videoId: number | string)
const video = await server.videos.get({ id: videoId })
const requests = [
makeRawRequest(join(server.url, video.thumbnailPath), HttpStatusCode.OK_200),
makeRawRequest(join(server.url, video.thumbnailPath), HttpStatusCode.OK_200)
makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }),
makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
]
for (const req of requests) {
@ -69,17 +69,17 @@ describe('Test regenerate thumbnails script', function () {
it('Should have empty thumbnails', async function () {
{
const res = await makeRawRequest(join(servers[0].url, video1.thumbnailPath), HttpStatusCode.OK_200)
const res = await makeGetRequest({ url: servers[0].url, path: video1.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
expect(res.body).to.have.lengthOf(0)
}
{
const res = await makeRawRequest(join(servers[0].url, video2.thumbnailPath), HttpStatusCode.OK_200)
const res = await makeGetRequest({ url: servers[0].url, path: video2.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
expect(res.body).to.not.have.lengthOf(0)
}
{
const res = await makeRawRequest(join(servers[0].url, remoteVideo.thumbnailPath), HttpStatusCode.OK_200)
const res = await makeGetRequest({ url: servers[0].url, path: remoteVideo.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
expect(res.body).to.have.lengthOf(0)
}
})
@ -94,21 +94,21 @@ describe('Test regenerate thumbnails script', function () {
await testThumbnail(servers[0], video1.uuid)
await testThumbnail(servers[0], video2.uuid)
const res = await makeRawRequest(join(servers[0].url, remoteVideo.thumbnailPath), HttpStatusCode.OK_200)
const res = await makeGetRequest({ url: servers[0].url, path: remoteVideo.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
expect(res.body).to.have.lengthOf(0)
})
it('Should have deleted old thumbnail files', async function () {
{
await makeRawRequest(join(servers[0].url, video1.thumbnailPath), HttpStatusCode.NOT_FOUND_404)
await makeGetRequest({ url: servers[0].url, path: video1.thumbnailPath, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
}
{
await makeRawRequest(join(servers[0].url, video2.thumbnailPath), HttpStatusCode.NOT_FOUND_404)
await makeGetRequest({ url: servers[0].url, path: video2.thumbnailPath, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
}
{
const res = await makeRawRequest(join(servers[0].url, remoteVideo.thumbnailPath), HttpStatusCode.OK_200)
const res = await makeGetRequest({ url: servers[0].url, path: remoteVideo.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
expect(res.body).to.have.lengthOf(0)
}
})

View File

@ -314,7 +314,7 @@ describe('Test syndication feeds', () => {
const jsonObj = JSON.parse(json)
const imageUrl = jsonObj.icon
expect(imageUrl).to.include('/lazy-static/avatars/')
await makeRawRequest(imageUrl)
await makeRawRequest({ url: imageUrl })
})
})

View File

@ -6,6 +6,7 @@ import {
cleanupTests,
createMultipleServers,
doubleFollow,
makeGetRequest,
makeRawRequest,
PeerTubeServer,
PluginsCommand,
@ -461,30 +462,41 @@ describe('Test plugin filter hooks', function () {
})
it('Should run filter:api.download.torrent.allowed.result', async function () {
const res = await makeRawRequest(downloadVideos[0].files[0].torrentDownloadUrl, 403)
const res = await makeRawRequest({ url: downloadVideos[0].files[0].torrentDownloadUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
expect(res.body.error).to.equal('Liu Bei')
await makeRawRequest(downloadVideos[1].files[0].torrentDownloadUrl, 200)
await makeRawRequest(downloadVideos[2].files[0].torrentDownloadUrl, 200)
await makeRawRequest({ url: downloadVideos[1].files[0].torrentDownloadUrl, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: downloadVideos[2].files[0].torrentDownloadUrl, expectedStatus: HttpStatusCode.OK_200 })
})
it('Should run filter:api.download.video.allowed.result', async function () {
{
const res = await makeRawRequest(downloadVideos[1].files[0].fileDownloadUrl, 403)
const res = await makeRawRequest({ url: downloadVideos[1].files[0].fileDownloadUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
expect(res.body.error).to.equal('Cao Cao')
await makeRawRequest(downloadVideos[0].files[0].fileDownloadUrl, 200)
await makeRawRequest(downloadVideos[2].files[0].fileDownloadUrl, 200)
await makeRawRequest({ url: downloadVideos[0].files[0].fileDownloadUrl, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: downloadVideos[2].files[0].fileDownloadUrl, expectedStatus: HttpStatusCode.OK_200 })
}
{
const res = await makeRawRequest(downloadVideos[2].streamingPlaylists[0].files[0].fileDownloadUrl, 403)
const res = await makeRawRequest({
url: downloadVideos[2].streamingPlaylists[0].files[0].fileDownloadUrl,
expectedStatus: HttpStatusCode.FORBIDDEN_403
})
expect(res.body.error).to.equal('Sun Jian')
await makeRawRequest(downloadVideos[2].files[0].fileDownloadUrl, 200)
await makeRawRequest({ url: downloadVideos[2].files[0].fileDownloadUrl, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest(downloadVideos[0].streamingPlaylists[0].files[0].fileDownloadUrl, 200)
await makeRawRequest(downloadVideos[1].streamingPlaylists[0].files[0].fileDownloadUrl, 200)
await makeRawRequest({
url: downloadVideos[0].streamingPlaylists[0].files[0].fileDownloadUrl,
expectedStatus: HttpStatusCode.OK_200
})
await makeRawRequest({
url: downloadVideos[1].streamingPlaylists[0].files[0].fileDownloadUrl,
expectedStatus: HttpStatusCode.OK_200
})
}
})
})
@ -515,12 +527,12 @@ describe('Test plugin filter hooks', function () {
})
it('Should run filter:html.embed.video.allowed.result', async function () {
const res = await makeRawRequest(servers[0].url + embedVideos[0].embedPath, 200)
const res = await makeGetRequest({ url: servers[0].url, path: embedVideos[0].embedPath, expectedStatus: HttpStatusCode.OK_200 })
expect(res.text).to.equal('Lu Bu')
})
it('Should run filter:html.embed.video-playlist.allowed.result', async function () {
const res = await makeRawRequest(servers[0].url + embedPlaylists[0].embedPath, 200)
const res = await makeGetRequest({ url: servers[0].url, path: embedPlaylists[0].embedPath, expectedStatus: HttpStatusCode.OK_200 })
expect(res.text).to.equal('Diao Chan')
})
})

View File

@ -307,7 +307,7 @@ describe('Test plugin helpers', function () {
expect(file.fps).to.equal(25)
expect(await pathExists(file.path)).to.be.true
await makeRawRequest(file.url, HttpStatusCode.OK_200)
await makeRawRequest({ url: file.url, expectedStatus: HttpStatusCode.OK_200 })
}
}
@ -321,12 +321,12 @@ describe('Test plugin helpers', function () {
const miniature = body.thumbnails.find(t => t.type === ThumbnailType.MINIATURE)
expect(miniature).to.exist
expect(await pathExists(miniature.path)).to.be.true
await makeRawRequest(miniature.url, HttpStatusCode.OK_200)
await makeRawRequest({ url: miniature.url, expectedStatus: HttpStatusCode.OK_200 })
const preview = body.thumbnails.find(t => t.type === ThumbnailType.PREVIEW)
expect(preview).to.exist
expect(await pathExists(preview.path)).to.be.true
await makeRawRequest(preview.url, HttpStatusCode.OK_200)
await makeRawRequest({ url: preview.url, expectedStatus: HttpStatusCode.OK_200 })
}
})

View File

@ -1,9 +1,13 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { expect } from 'chai'
import { basename } from 'path'
import { removeFragmentedMP4Ext } from '@shared/core-utils'
import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils'
import { sha256 } from '@shared/extra-utils'
import { HttpStatusCode, VideoStreamingPlaylist } from '@shared/models'
import { PeerTubeServer } from '@shared/server-commands'
import { HttpStatusCode, VideoStreamingPlaylist, VideoStreamingPlaylistType } from '@shared/models'
import { makeRawRequest, PeerTubeServer, webtorrentAdd } from '@shared/server-commands'
import { expectStartWith } from './checks'
import { hlsInfohashExist } from './tracker'
async function checkSegmentHash (options: {
server: PeerTubeServer
@ -75,8 +79,118 @@ async function checkResolutionsInMasterPlaylist (options: {
expect(playlistsLength).to.have.lengthOf(resolutions.length)
}
async function completeCheckHlsPlaylist (options: {
servers: PeerTubeServer[]
videoUUID: string
hlsOnly: boolean
resolutions?: number[]
objectStorageBaseUrl: string
}) {
const { videoUUID, hlsOnly, objectStorageBaseUrl } = options
const resolutions = options.resolutions ?? [ 240, 360, 480, 720 ]
for (const server of options.servers) {
const videoDetails = await server.videos.get({ id: videoUUID })
const baseUrl = `http://${videoDetails.account.host}`
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
const hlsPlaylist = videoDetails.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
expect(hlsPlaylist).to.not.be.undefined
const hlsFiles = hlsPlaylist.files
expect(hlsFiles).to.have.lengthOf(resolutions.length)
if (hlsOnly) expect(videoDetails.files).to.have.lengthOf(0)
else expect(videoDetails.files).to.have.lengthOf(resolutions.length)
// Check JSON files
for (const resolution of resolutions) {
const file = hlsFiles.find(f => f.resolution.id === resolution)
expect(file).to.not.be.undefined
expect(file.magnetUri).to.have.lengthOf.above(2)
expect(file.torrentUrl).to.match(
new RegExp(`http://${server.host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}-hls.torrent`)
)
if (objectStorageBaseUrl) {
expectStartWith(file.fileUrl, objectStorageBaseUrl)
} else {
expect(file.fileUrl).to.match(
new RegExp(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${uuidRegex}-${file.resolution.id}-fragmented.mp4`)
)
}
expect(file.resolution.label).to.equal(resolution + 'p')
await makeRawRequest({ url: file.torrentUrl, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
const torrent = await webtorrentAdd(file.magnetUri, true)
expect(torrent.files).to.be.an('array')
expect(torrent.files.length).to.equal(1)
expect(torrent.files[0].path).to.exist.and.to.not.equal('')
}
// Check master playlist
{
await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions })
const masterPlaylist = await server.streamingPlaylists.get({ url: hlsPlaylist.playlistUrl })
let i = 0
for (const resolution of resolutions) {
expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
const url = 'http://' + videoDetails.account.host
await hlsInfohashExist(url, hlsPlaylist.playlistUrl, i)
i++
}
}
// Check resolution playlists
{
for (const resolution of resolutions) {
const file = hlsFiles.find(f => f.resolution.id === resolution)
const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8'
const url = objectStorageBaseUrl
? `${objectStorageBaseUrl}hls/${videoUUID}/${playlistName}`
: `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}`
const subPlaylist = await server.streamingPlaylists.get({ url })
expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`))
expect(subPlaylist).to.contain(basename(file.fileUrl))
}
}
{
const baseUrlAndPath = objectStorageBaseUrl
? objectStorageBaseUrl + 'hls/' + videoUUID
: baseUrl + '/static/streaming-playlists/hls/' + videoUUID
for (const resolution of resolutions) {
await checkSegmentHash({
server,
baseUrlPlaylist: baseUrlAndPath,
baseUrlSegment: baseUrlAndPath,
resolution,
hlsPlaylist
})
}
}
}
}
export {
checkSegmentHash,
checkLiveSegmentHash,
checkResolutionsInMasterPlaylist
checkResolutionsInMasterPlaylist,
completeCheckHlsPlaylist
}

View File

@ -125,9 +125,9 @@ async function completeVideoCheck (
expect(file.fileDownloadUrl).to.match(new RegExp(`http://${originHost}/download/videos/${uuidRegex}-${file.resolution.id}${extension}`))
await Promise.all([
makeRawRequest(file.torrentUrl, 200),
makeRawRequest(file.torrentDownloadUrl, 200),
makeRawRequest(file.metadataUrl, 200)
makeRawRequest({ url: file.torrentUrl, expectedStatus: HttpStatusCode.OK_200 }),
makeRawRequest({ url: file.torrentDownloadUrl, expectedStatus: HttpStatusCode.OK_200 }),
makeRawRequest({ url: file.metadataUrl, expectedStatus: HttpStatusCode.OK_200 })
])
expect(file.resolution.id).to.equal(attributeFile.resolution)

View File

@ -1,6 +1,16 @@
import { Video, VideoPlaylist } from '../../models'
import { secondsToTime } from './date'
function addQueryParams (url: string, params: { [ id: string ]: string }) {
const objUrl = new URL(url)
for (const key of Object.keys(params)) {
objUrl.searchParams.append(key, params[key])
}
return objUrl.toString()
}
function buildPlaylistLink (playlist: Pick<VideoPlaylist, 'shortUUID'>, base?: string) {
return (base ?? window.location.origin) + buildPlaylistWatchPath(playlist)
}
@ -103,6 +113,8 @@ function decoratePlaylistLink (options: {
// ---------------------------------------------------------------------------
export {
addQueryParams,
buildPlaylistLink,
buildVideoLink,

View File

@ -8,4 +8,5 @@ export interface SendDebugCommand {
| 'process-video-views-buffer'
| 'process-video-viewers'
| 'process-video-channel-sync-latest'
| 'process-update-videos-scheduler'
}

View File

@ -33,6 +33,8 @@ export * from './video-storage.enum'
export * from './video-streaming-playlist.model'
export * from './video-streaming-playlist.type'
export * from './video-token.model'
export * from './video-update.model'
export * from './video-view.model'
export * from './video.model'

View File

@ -0,0 +1,6 @@
export interface VideoToken {
files: {
token: string
expires: string | Date
}
}

View File

@ -3,7 +3,7 @@
import { decode } from 'querystring'
import request from 'supertest'
import { URL } from 'url'
import { buildAbsoluteFixturePath } from '@shared/core-utils'
import { buildAbsoluteFixturePath, pick } from '@shared/core-utils'
import { HttpStatusCode } from '@shared/models'
export type CommonRequestParams = {
@ -21,10 +21,21 @@ export type CommonRequestParams = {
expectedStatus?: HttpStatusCode
}
function makeRawRequest (url: string, expectedStatus?: HttpStatusCode, range?: string) {
const { host, protocol, pathname } = new URL(url)
function makeRawRequest (options: {
url: string
token?: string
expectedStatus?: HttpStatusCode
range?: string
query?: { [ id: string ]: string }
}) {
const { host, protocol, pathname } = new URL(options.url)
return makeGetRequest({ url: `${protocol}//${host}`, path: pathname, expectedStatus, range })
return makeGetRequest({
url: `${protocol}//${host}`,
path: pathname,
...pick(options, [ 'expectedStatus', 'range', 'token', 'query' ])
})
}
function makeGetRequest (options: CommonRequestParams & {

View File

@ -36,6 +36,7 @@ import {
StreamingPlaylistsCommand,
VideosCommand,
VideoStudioCommand,
VideoTokenCommand,
ViewsCommand
} from '../videos'
import { CommentsCommand } from '../videos/comments-command'
@ -145,6 +146,7 @@ export class PeerTubeServer {
videoStats?: VideoStatsCommand
views?: ViewsCommand
twoFactor?: TwoFactorCommand
videoToken?: VideoTokenCommand
constructor (options: { serverNumber: number } | { url: string }) {
if ((options as any).url) {
@ -427,5 +429,6 @@ export class PeerTubeServer {
this.videoStats = new VideoStatsCommand(this)
this.views = new ViewsCommand(this)
this.twoFactor = new TwoFactorCommand(this)
this.videoToken = new VideoTokenCommand(this)
}
}

View File

@ -14,5 +14,6 @@ export * from './services-command'
export * from './streaming-playlists-command'
export * from './comments-command'
export * from './video-studio-command'
export * from './video-token-command'
export * from './views-command'
export * from './videos-command'

View File

@ -12,6 +12,7 @@ import {
ResultList,
VideoCreateResult,
VideoDetails,
VideoPrivacy,
VideoState
} from '@shared/models'
import { unwrapBody } from '../requests'
@ -115,6 +116,31 @@ export class LiveCommand extends AbstractCommand {
return body.video
}
async quickCreate (options: OverrideCommandOptions & {
saveReplay: boolean
permanentLive: boolean
privacy?: VideoPrivacy
}) {
const { saveReplay, permanentLive, privacy } = options
const { uuid } = await this.create({
...options,
fields: {
name: 'live',
permanentLive,
saveReplay,
channelId: this.server.store.channel.id,
privacy
}
})
const video = await this.server.videos.getWithToken({ id: uuid })
const live = await this.get({ videoId: uuid })
return { video, live }
}
// ---------------------------------------------------------------------------
async sendRTMPStreamInVideo (options: OverrideCommandOptions & {

View File

@ -1,6 +1,6 @@
import ffmpeg, { FfmpegCommand } from 'fluent-ffmpeg'
import { buildAbsoluteFixturePath, wait } from '@shared/core-utils'
import { VideoDetails, VideoInclude } from '@shared/models'
import { VideoDetails, VideoInclude, VideoPrivacy } from '@shared/models'
import { PeerTubeServer } from '../server/server'
function sendRTMPStream (options: {
@ -98,7 +98,10 @@ async function waitUntilLiveReplacedByReplayOnAllServers (servers: PeerTubeServe
}
async function findExternalSavedVideo (server: PeerTubeServer, liveDetails: VideoDetails) {
const { data } = await server.videos.list({ token: server.accessToken, sort: '-publishedAt', include: VideoInclude.BLACKLISTED })
const include = VideoInclude.BLACKLISTED
const privacyOneOf = [ VideoPrivacy.INTERNAL, VideoPrivacy.PRIVATE, VideoPrivacy.PUBLIC, VideoPrivacy.UNLISTED ]
const { data } = await server.videos.list({ token: server.accessToken, sort: '-publishedAt', include, privacyOneOf })
return data.find(v => v.name === liveDetails.name + ' - ' + new Date(liveDetails.publishedAt).toLocaleString())
}

Some files were not shown because too many files have changed in this diff Show More