Resumable video uploads (#3933)

* WIP: resumable video uploads

relates to #324

* fix review comments

* video upload: error handling

* fix audio upload

* fixes after self review

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/middlewares/validators/videos/videos.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* update after code review

* refactor upload route

- restore multipart upload route
- move resumable to dedicated upload-resumable route
- move checks to middleware
- do not leak internal fs structure in response

* fix yarn.lock upon rebase

* factorize addVideo for reuse in both endpoints

* add resumable upload API to openapi spec

* add initial test and test helper for resumable upload

* typings for videoAddResumable middleware

* avoid including aws and google packages via node-uploadx, by only including uploadx/core

* rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio

* add video-upload-tmp-folder-cleaner job

* stronger typing of video upload middleware

* reduce dependency to @uploadx/core

* add audio upload test

* refactor resumable uploads cleanup from job to scheduler

* refactor resumable uploads scheduler to compare to last execution time

* make resumable upload validator to always cleanup on failure

* move legacy upload request building outside of uploadVideo test helper

* filter upload-resumable middlewares down to POST, PUT, DELETE

also begin to type metadata

* merge add duration functions

* stronger typings and documentation for uploadx behaviour, move init validator up

* refactor(client/video-edit): options > uploadxOptions

* refactor(client/video-edit): remove obsolete else

* scheduler/remove-dangling-resum: rename tag

* refactor(server/video): add UploadVideoFiles type

* refactor(mw/validators): restructure eslint disable

* refactor(mw/validators/videos): rename import

* refactor(client/vid-upload): rename html elem id

* refactor(sched/remove-dangl): move fn to method

* refactor(mw/async): add method typing

* refactor(mw/vali/video): double quote > single

* refactor(server/upload-resum): express use > all

* proper http methud enum server/middlewares/async.ts

* properly type http methods

* factorize common video upload validation steps

* add check for maximum partially uploaded file size

* fix audioBg use

* fix extname(filename) in addVideo

* document parameters for uploadx's resumable protocol

* clear META files in scheduler

* last audio refactor before cramming preview in the initial POST form data

* refactor as mulitpart/form-data initial post request

this allows preview/thumbnail uploads alongside the initial request,
and cleans up the upload form

* Add more tests for resumable uploads

* Refactor remove dangling resumable uploads

* Prepare changelog

* Add more resumable upload tests

* Remove user quota check for resumable uploads

* Fix upload error handler

* Update nginx template for upload-resumable

* Cleanup comment

* Remove unused express methods

* Prefer to use got instead of raw http

* Don't retry on error 500

Co-authored-by: Rigel Kent <par@rigelk.eu>
Co-authored-by: Rigel Kent <sendmemail@rigelk.eu>
Co-authored-by: Chocobozzz <me@florianbigard.com>
This commit is contained in:
kontrollanten 2021-05-10 11:13:41 +02:00 committed by GitHub
parent d29ced1a85
commit f6d6e7f861
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
46 changed files with 2454 additions and 1293 deletions

View File

@ -96,6 +96,7 @@
"lodash-es": "^4.17.4",
"markdown-it": "12.0.4",
"mini-css-extract-plugin": "^1.3.1",
"ngx-uploadx": "^4.1.0",
"p2p-media-loader-hlsjs": "^0.6.2",
"path-browserify": "^1.0.0",
"primeng": "^11.0.0-rc.1",

View File

@ -2,7 +2,7 @@ import { ViewportScroller } from '@angular/common'
import { HttpErrorResponse } from '@angular/common/http'
import { AfterViewChecked, Component, OnInit } from '@angular/core'
import { AuthService, Notifier, User, UserService } from '@app/core'
import { uploadErrorHandler } from '@app/helpers'
import { genericUploadErrorHandler } from '@app/helpers'
@Component({
selector: 'my-account-settings',
@ -46,7 +46,7 @@ export class MyAccountSettingsComponent implements OnInit, AfterViewChecked {
this.user.updateAccountAvatar(data.avatar)
},
(err: HttpErrorResponse) => uploadErrorHandler({
(err: HttpErrorResponse) => genericUploadErrorHandler({
err,
name: $localize`avatar`,
notifier: this.notifier

View File

@ -3,7 +3,7 @@ import { HttpErrorResponse } from '@angular/common/http'
import { Component, OnDestroy, OnInit } from '@angular/core'
import { ActivatedRoute, Router } from '@angular/router'
import { AuthService, Notifier, ServerService } from '@app/core'
import { uploadErrorHandler } from '@app/helpers'
import { genericUploadErrorHandler } from '@app/helpers'
import {
VIDEO_CHANNEL_DESCRIPTION_VALIDATOR,
VIDEO_CHANNEL_DISPLAY_NAME_VALIDATOR,
@ -109,7 +109,7 @@ export class MyVideoChannelUpdateComponent extends MyVideoChannelEdit implements
this.videoChannel.updateAvatar(data.avatar)
},
(err: HttpErrorResponse) => uploadErrorHandler({
(err: HttpErrorResponse) => genericUploadErrorHandler({
err,
name: $localize`avatar`,
notifier: this.notifier
@ -139,7 +139,7 @@ export class MyVideoChannelUpdateComponent extends MyVideoChannelEdit implements
this.videoChannel.updateBanner(data.banner)
},
(err: HttpErrorResponse) => uploadErrorHandler({
(err: HttpErrorResponse) => genericUploadErrorHandler({
err,
name: $localize`banner`,
notifier: this.notifier

View File

@ -0,0 +1,48 @@
import { objectToFormData } from '@app/helpers'
import { resolveUrl, UploaderX } from 'ngx-uploadx'
/**
* multipart/form-data uploader extending the UploaderX implementation of Google Resumable
* for use with multer
*
* @see https://github.com/kukhariev/ngx-uploadx/blob/637e258fe366b8095203f387a6101a230ee4f8e6/src/uploadx/lib/uploaderx.ts
* @example
*
* options: UploadxOptions = {
* uploaderClass: UploaderXFormData
* };
*/
export class UploaderXFormData extends UploaderX {
async getFileUrl (): Promise<string> {
const headers = {
'X-Upload-Content-Length': this.size.toString(),
'X-Upload-Content-Type': this.file.type || 'application/octet-stream'
}
const previewfile = this.metadata.previewfile as any as File
delete this.metadata.previewfile
const data = objectToFormData(this.metadata)
if (previewfile !== undefined) {
data.append('previewfile', previewfile, previewfile.name)
data.append('thumbnailfile', previewfile, previewfile.name)
}
await this.request({
method: 'POST',
body: data,
url: this.endpoint,
headers
})
const location = this.getValueFromResponse('location')
if (!location) {
throw new Error('Invalid or missing Location header')
}
this.offset = this.responseStatus === 201 ? 0 : undefined
return resolveUrl(location, this.endpoint)
}
}

View File

@ -1,12 +1,17 @@
<div *ngIf="!isUploadingVideo" class="upload-video-container" dragDrop (fileDropped)="setVideoFile($event)">
<div *ngIf="!isUploadingVideo" class="upload-video-container" dragDrop (fileDropped)="onFileDropped($event)">
<div class="first-step-block">
<my-global-icon class="upload-icon" iconName="upload" aria-hidden="true"></my-global-icon>
<div class="button-file form-control" [ngbTooltip]="'(extensions: ' + videoExtensions + ')'">
<span i18n>Select the file to upload</span>
<input
aria-label="Select the file to upload" i18n-aria-label
#videofileInput type="file" name="videofile" id="videofile" [accept]="videoExtensions" (change)="fileChange()" autofocus
aria-label="Select the file to upload"
i18n-aria-label
#videofileInput
[accept]="videoExtensions"
(change)="onFileChange($event)"
id="videofile"
type="file"
/>
</div>
@ -41,7 +46,13 @@
</div>
<div class="form-group upload-audio-button">
<my-button className="orange-button" i18n-label [label]="getAudioUploadLabel()" icon="upload" (click)="uploadFirstStep(true)"></my-button>
<my-button
className="orange-button"
[label]="getAudioUploadLabel()"
icon="upload"
(click)="uploadAudio()"
>
</my-button>
</div>
</ng-container>
</div>
@ -64,6 +75,7 @@
<span>{{ error }}</span>
</div>
</div>
<div class="btn-group" role="group">
<input type="button" class="btn" i18n-value="Retry failed upload of a video" value="Retry" (click)="retryUpload()" />
<input type="button" class="btn" i18n-value="Cancel ongoing upload of a video" value="Cancel" (click)="cancelUpload()" />

View File

@ -47,8 +47,4 @@
margin-left: 10px;
}
.btn-group > input:not(:first-child) {
margin-left: 0;
}
}

View File

@ -1,15 +1,16 @@
import { Subscription } from 'rxjs'
import { HttpErrorResponse, HttpEventType, HttpResponse } from '@angular/common/http'
import { AfterViewInit, Component, ElementRef, EventEmitter, OnDestroy, OnInit, Output, ViewChild } from '@angular/core'
import { Router } from '@angular/router'
import { UploadxOptions, UploadState, UploadxService } from 'ngx-uploadx'
import { UploaderXFormData } from './uploaderx-form-data'
import { AuthService, CanComponentDeactivate, HooksService, Notifier, ServerService, UserService } from '@app/core'
import { scrollToTop, uploadErrorHandler } from '@app/helpers'
import { scrollToTop, genericUploadErrorHandler } from '@app/helpers'
import { FormValidatorService } from '@app/shared/shared-forms'
import { BytesPipe, VideoCaptionService, VideoEdit, VideoService } from '@app/shared/shared-main'
import { LoadingBarService } from '@ngx-loading-bar/core'
import { HttpStatusCode } from '@shared/core-utils/miscs/http-error-codes'
import { VideoPrivacy } from '@shared/models'
import { VideoSend } from './video-send'
import { HttpErrorResponse, HttpEventType, HttpHeaders } from '@angular/common/http'
@Component({
selector: 'my-video-upload',
@ -20,23 +21,18 @@ import { VideoSend } from './video-send'
'./video-send.scss'
]
})
export class VideoUploadComponent extends VideoSend implements OnInit, AfterViewInit, OnDestroy, CanComponentDeactivate {
export class VideoUploadComponent extends VideoSend implements OnInit, OnDestroy, AfterViewInit, CanComponentDeactivate {
@Output() firstStepDone = new EventEmitter<string>()
@Output() firstStepError = new EventEmitter<void>()
@ViewChild('videofileInput') videofileInput: ElementRef<HTMLInputElement>
// So that it can be accessed in the template
readonly SPECIAL_SCHEDULED_PRIVACY = VideoEdit.SPECIAL_SCHEDULED_PRIVACY
userVideoQuotaUsed = 0
userVideoQuotaUsedDaily = 0
isUploadingAudioFile = false
isUploadingVideo = false
isUpdatingVideo = false
videoUploaded = false
videoUploadObservable: Subscription = null
videoUploadPercents = 0
videoUploadedIds = {
id: 0,
@ -49,7 +45,13 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView
error: string
enableRetryAfterError: boolean
// So that it can be accessed in the template
protected readonly DEFAULT_VIDEO_PRIVACY = VideoPrivacy.PUBLIC
protected readonly BASE_VIDEO_UPLOAD_URL = VideoService.BASE_VIDEO_URL + 'upload-resumable'
private uploadxOptions: UploadxOptions
private isUpdatingVideo = false
private fileToUpload: File
constructor (
protected formValidatorService: FormValidatorService,
@ -61,15 +63,77 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView
protected videoCaptionService: VideoCaptionService,
private userService: UserService,
private router: Router,
private hooks: HooksService
) {
private hooks: HooksService,
private resumableUploadService: UploadxService
) {
super()
this.uploadxOptions = {
endpoint: this.BASE_VIDEO_UPLOAD_URL,
multiple: false,
token: this.authService.getAccessToken(),
uploaderClass: UploaderXFormData,
retryConfig: {
maxAttempts: 6,
shouldRetry: (code: number) => {
return code < 400 || code >= 501
}
}
}
}
get videoExtensions () {
return this.serverConfig.video.file.extensions.join(', ')
}
onUploadVideoOngoing (state: UploadState) {
switch (state.status) {
case 'error':
const error = state.response?.error || 'Unknow error'
this.handleUploadError({
error: new Error(error),
name: 'HttpErrorResponse',
message: error,
ok: false,
headers: new HttpHeaders(state.responseHeaders),
status: +state.responseStatus,
statusText: error,
type: HttpEventType.Response,
url: state.url
})
break
case 'cancelled':
this.isUploadingVideo = false
this.videoUploadPercents = 0
this.firstStepError.emit()
this.enableRetryAfterError = false
this.error = ''
break
case 'queue':
this.closeFirstStep(state.name)
break
case 'uploading':
this.videoUploadPercents = state.progress
break
case 'paused':
this.notifier.info($localize`Upload cancelled`)
break
case 'complete':
this.videoUploaded = true
this.videoUploadPercents = 100
this.videoUploadedIds = state?.response.video
break
}
}
ngOnInit () {
super.ngOnInit()
@ -78,6 +142,9 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView
this.userVideoQuotaUsed = data.videoQuotaUsed
this.userVideoQuotaUsedDaily = data.videoQuotaUsedDaily
})
this.resumableUploadService.events
.subscribe(state => this.onUploadVideoOngoing(state))
}
ngAfterViewInit () {
@ -85,7 +152,7 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView
}
ngOnDestroy () {
if (this.videoUploadObservable) this.videoUploadObservable.unsubscribe()
this.cancelUpload()
}
canDeactivate () {
@ -105,137 +172,43 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView
}
}
getVideoFile () {
return this.videofileInput.nativeElement.files[0]
}
setVideoFile (files: FileList) {
onFileDropped (files: FileList) {
this.videofileInput.nativeElement.files = files
this.fileChange()
this.onFileChange({ target: this.videofileInput.nativeElement })
}
getAudioUploadLabel () {
const videofile = this.getVideoFile()
if (!videofile) return $localize`Upload`
onFileChange (event: Event | { target: HTMLInputElement }) {
const file = (event.target as HTMLInputElement).files[0]
return $localize`Upload ${videofile.name}`
if (!file) return
if (!this.checkGlobalUserQuota(file)) return
if (!this.checkDailyUserQuota(file)) return
if (this.isAudioFile(file.name)) {
this.isUploadingAudioFile = true
return
}
this.isUploadingVideo = true
this.fileToUpload = file
this.uploadFile(file)
}
fileChange () {
this.uploadFirstStep()
uploadAudio () {
this.uploadFile(this.getInputVideoFile(), this.previewfileUpload)
}
retryUpload () {
this.enableRetryAfterError = false
this.error = ''
this.uploadVideo()
this.uploadFile(this.fileToUpload)
}
cancelUpload () {
if (this.videoUploadObservable !== null) {
this.videoUploadObservable.unsubscribe()
}
this.isUploadingVideo = false
this.videoUploadPercents = 0
this.videoUploadObservable = null
this.firstStepError.emit()
this.enableRetryAfterError = false
this.error = ''
this.notifier.info($localize`Upload cancelled`)
}
uploadFirstStep (clickedOnButton = false) {
const videofile = this.getVideoFile()
if (!videofile) return
if (!this.checkGlobalUserQuota(videofile)) return
if (!this.checkDailyUserQuota(videofile)) return
if (clickedOnButton === false && this.isAudioFile(videofile.name)) {
this.isUploadingAudioFile = true
return
}
// Build name field
const nameWithoutExtension = videofile.name.replace(/\.[^/.]+$/, '')
let name: string
// If the name of the file is very small, keep the extension
if (nameWithoutExtension.length < 3) name = videofile.name
else name = nameWithoutExtension
const nsfw = this.serverConfig.instance.isNSFW
const waitTranscoding = true
const commentsEnabled = true
const downloadEnabled = true
const channelId = this.firstStepChannelId.toString()
this.formData = new FormData()
this.formData.append('name', name)
// Put the video "private" -> we are waiting the user validation of the second step
this.formData.append('privacy', VideoPrivacy.PRIVATE.toString())
this.formData.append('nsfw', '' + nsfw)
this.formData.append('commentsEnabled', '' + commentsEnabled)
this.formData.append('downloadEnabled', '' + downloadEnabled)
this.formData.append('waitTranscoding', '' + waitTranscoding)
this.formData.append('channelId', '' + channelId)
this.formData.append('videofile', videofile)
if (this.previewfileUpload) {
this.formData.append('previewfile', this.previewfileUpload)
this.formData.append('thumbnailfile', this.previewfileUpload)
}
this.isUploadingVideo = true
this.firstStepDone.emit(name)
this.form.patchValue({
name,
privacy: this.firstStepPrivacyId,
nsfw,
channelId: this.firstStepChannelId,
previewfile: this.previewfileUpload
})
this.uploadVideo()
}
uploadVideo () {
this.videoUploadObservable = this.videoService.uploadVideo(this.formData).subscribe(
event => {
if (event.type === HttpEventType.UploadProgress) {
this.videoUploadPercents = Math.round(100 * event.loaded / event.total)
} else if (event instanceof HttpResponse) {
this.videoUploaded = true
this.videoUploadedIds = event.body.video
this.videoUploadObservable = null
}
},
(err: HttpErrorResponse) => {
// Reset progress (but keep isUploadingVideo true)
this.videoUploadPercents = 0
this.videoUploadObservable = null
this.enableRetryAfterError = true
this.error = uploadErrorHandler({
err,
name: $localize`video`,
notifier: this.notifier,
sticky: false
})
if (err.status === HttpStatusCode.PAYLOAD_TOO_LARGE_413 ||
err.status === HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415) {
this.cancelUpload()
}
}
)
this.resumableUploadService.control({ action: 'cancel' })
}
isPublishingButtonDisabled () {
@ -245,6 +218,13 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView
!this.videoUploadedIds.id
}
getAudioUploadLabel () {
const videofile = this.getInputVideoFile()
if (!videofile) return $localize`Upload`
return $localize`Upload ${videofile.name}`
}
updateSecondStep () {
if (this.isPublishingButtonDisabled() || !this.checkForm()) {
return
@ -275,6 +255,62 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView
)
}
private getInputVideoFile () {
return this.videofileInput.nativeElement.files[0]
}
private uploadFile (file: File, previewfile?: File) {
const metadata = {
waitTranscoding: true,
commentsEnabled: true,
downloadEnabled: true,
channelId: this.firstStepChannelId,
nsfw: this.serverConfig.instance.isNSFW,
privacy: VideoPrivacy.PRIVATE.toString(),
filename: file.name,
previewfile: previewfile as any
}
this.resumableUploadService.handleFiles(file, {
...this.uploadxOptions,
metadata
})
this.isUploadingVideo = true
}
private handleUploadError (err: HttpErrorResponse) {
// Reset progress (but keep isUploadingVideo true)
this.videoUploadPercents = 0
this.enableRetryAfterError = true
this.error = genericUploadErrorHandler({
err,
name: $localize`video`,
notifier: this.notifier,
sticky: false
})
if (err.status === HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415) {
this.cancelUpload()
}
}
private closeFirstStep (filename: string) {
const nameWithoutExtension = filename.replace(/\.[^/.]+$/, '')
const name = nameWithoutExtension.length < 3 ? filename : nameWithoutExtension
this.form.patchValue({
name,
privacy: this.firstStepPrivacyId,
nsfw: this.serverConfig.instance.isNSFW,
channelId: this.firstStepChannelId,
previewfile: this.previewfileUpload
})
this.firstStepDone.emit(name)
}
private checkGlobalUserQuota (videofile: File) {
const bytePipes = new BytesPipe()
@ -285,8 +321,7 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView
const videoQuotaUsedBytes = bytePipes.transform(this.userVideoQuotaUsed, 0)
const videoQuotaBytes = bytePipes.transform(videoQuota, 0)
const msg = $localize`Your video quota is exceeded with this video (
video size: ${videoSizeBytes}, used: ${videoQuotaUsedBytes}, quota: ${videoQuotaBytes})`
const msg = $localize`Your video quota is exceeded with this video (video size: ${videoSizeBytes}, used: ${videoQuotaUsedBytes}, quota: ${videoQuotaBytes})`
this.notifier.error(msg)
return false
@ -304,9 +339,7 @@ video size: ${videoSizeBytes}, used: ${videoQuotaUsedBytes}, quota: ${videoQuota
const videoSizeBytes = bytePipes.transform(videofile.size, 0)
const quotaUsedDailyBytes = bytePipes.transform(this.userVideoQuotaUsedDaily, 0)
const quotaDailyBytes = bytePipes.transform(videoQuotaDaily, 0)
const msg = $localize`Your daily video quota is exceeded with this video (
video size: ${videoSizeBytes}, used: ${quotaUsedDailyBytes}, quota: ${quotaDailyBytes})`
const msg = $localize`Your daily video quota is exceeded with this video (video size: ${videoSizeBytes}, used: ${quotaUsedDailyBytes}, quota: ${quotaDailyBytes})`
this.notifier.error(msg)
return false

View File

@ -1,5 +1,6 @@
import { NgModule } from '@angular/core'
import { CanDeactivateGuard } from '@app/core'
import { UploadxModule } from 'ngx-uploadx'
import { VideoEditModule } from './shared/video-edit.module'
import { DragDropDirective } from './video-add-components/drag-drop.directive'
import { VideoImportTorrentComponent } from './video-add-components/video-import-torrent.component'
@ -13,7 +14,9 @@ import { VideoAddComponent } from './video-add.component'
imports: [
VideoAddRoutingModule,
VideoEditModule
VideoEditModule,
UploadxModule
],
declarations: [

View File

@ -173,8 +173,8 @@ function isXPercentInViewport (el: HTMLElement, percentVisible: number) {
)
}
function uploadErrorHandler (parameters: {
err: HttpErrorResponse
function genericUploadErrorHandler (parameters: {
err: Pick<HttpErrorResponse, 'message' | 'status' | 'headers'>
name: string
notifier: Notifier
sticky?: boolean
@ -186,6 +186,9 @@ function uploadErrorHandler (parameters: {
if (err instanceof ErrorEvent) { // network error
message = $localize`The connection was interrupted`
notifier.error(message, title, null, sticky)
} else if (err.status === HttpStatusCode.INTERNAL_SERVER_ERROR_500) {
message = $localize`The server encountered an error`
notifier.error(message, title, null, sticky)
} else if (err.status === HttpStatusCode.REQUEST_TIMEOUT_408) {
message = $localize`Your ${name} file couldn't be transferred before the set timeout (usually 10min)`
notifier.error(message, title, null, sticky)
@ -216,5 +219,5 @@ export {
isInViewport,
isXPercentInViewport,
listUserChannels,
uploadErrorHandler
genericUploadErrorHandler
}

View File

@ -7793,6 +7793,13 @@ next-tick@~1.0.0:
resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.0.0.tgz#ca86d1fe8828169b0120208e3dc8424b9db8342c"
integrity sha1-yobR/ogoFpsBICCOPchCS524NCw=
ngx-uploadx@^4.1.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/ngx-uploadx/-/ngx-uploadx-4.1.0.tgz#b3ed4566a2505239026bbdc10c2345aae28d67df"
integrity sha512-KCG0NT4SBc/5MRl8aR6joHHg+WeTdrkhLeC1DrNgVxrTBuuenlEwOVDpkLJMPX/8HE6Bq33rx1U2NNZYVl9NMQ==
dependencies:
tslib "^1.9.0"
nice-try@^1.0.4:
version "1.0.5"
resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366"

View File

@ -73,6 +73,7 @@
"swagger-cli": "swagger-cli"
},
"dependencies": {
"@uploadx/core": "^4.4.0",
"apicache": "1.6.2",
"async": "^3.0.1",
"async-lru": "^1.1.1",

View File

@ -116,6 +116,7 @@ import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-upd
import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler'
import { RemoveOldHistoryScheduler } from './server/lib/schedulers/remove-old-history-scheduler'
import { AutoFollowIndexInstances } from './server/lib/schedulers/auto-follow-index-instances'
import { RemoveDanglingResumableUploadsScheduler } from './server/lib/schedulers/remove-dangling-resumable-uploads-scheduler'
import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto'
import { PeerTubeSocket } from './server/lib/peertube-socket'
import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls'
@ -280,6 +281,7 @@ async function startApplication () {
PluginsCheckScheduler.Instance.enable()
PeerTubeVersionCheckScheduler.Instance.enable()
AutoFollowIndexInstances.Instance.enable()
RemoveDanglingResumableUploadsScheduler.Instance.enable()
// Redis initialization
Redis.Instance.init()

View File

@ -1,4 +1,6 @@
import { InboxManager } from '@server/lib/activitypub/inbox-manager'
import { RemoveDanglingResumableUploadsScheduler } from '@server/lib/schedulers/remove-dangling-resumable-uploads-scheduler'
import { SendDebugCommand } from '@shared/models'
import * as express from 'express'
import { UserRight } from '../../../../shared/models/users'
import { authenticate, ensureUserHasRight } from '../../../middlewares'
@ -11,6 +13,12 @@ debugRouter.get('/debug',
getDebug
)
debugRouter.post('/debug/run-command',
authenticate,
ensureUserHasRight(UserRight.MANAGE_DEBUG),
runCommand
)
// ---------------------------------------------------------------------------
export {
@ -25,3 +33,13 @@ function getDebug (req: express.Request, res: express.Response) {
activityPubMessagesWaiting: InboxManager.Instance.getActivityPubMessagesWaiting()
})
}
async function runCommand (req: express.Request, res: express.Response) {
const body: SendDebugCommand = req.body
if (body.command === 'remove-dandling-resumable-uploads') {
await RemoveDanglingResumableUploadsScheduler.Instance.execute()
}
return res.sendStatus(204)
}

View File

@ -2,6 +2,7 @@ import * as express from 'express'
import { move } from 'fs-extra'
import { extname } from 'path'
import toInt from 'validator/lib/toInt'
import { deleteResumableUploadMetaFile, getResumableUploadPath } from '@server/helpers/upload'
import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
import { changeVideoChannelShare } from '@server/lib/activitypub/share'
import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
@ -10,8 +11,9 @@ import { addOptimizeOrMergeAudioJob, buildLocalVideoFromReq, buildVideoThumbnail
import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths'
import { getServerActor } from '@server/models/application/application'
import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
import { uploadx } from '@uploadx/core'
import { VideoCreate, VideosCommonQuery, VideoState, VideoUpdate } from '../../../../shared'
import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
import { HttpStatusCode } from '../../../../shared/core-utils/miscs'
import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils'
import { buildNSFWFilter, createReqFiles, getCountVideos } from '../../../helpers/express-utils'
@ -47,7 +49,9 @@ import {
setDefaultPagination,
setDefaultVideosSort,
videoFileMetadataGetValidator,
videosAddValidator,
videosAddLegacyValidator,
videosAddResumableInitValidator,
videosAddResumableValidator,
videosCustomGetValidator,
videosGetValidator,
videosRemoveValidator,
@ -69,6 +73,7 @@ import { watchingRouter } from './watching'
const lTags = loggerTagsFactory('api', 'video')
const auditLogger = auditLoggerFactory('videos')
const videosRouter = express.Router()
const uploadxMiddleware = uploadx.upload({ directory: getResumableUploadPath() })
const reqVideoFileAdd = createReqFiles(
[ 'videofile', 'thumbnailfile', 'previewfile' ],
@ -79,6 +84,16 @@ const reqVideoFileAdd = createReqFiles(
previewfile: CONFIG.STORAGE.TMP_DIR
}
)
const reqVideoFileAddResumable = createReqFiles(
[ 'thumbnailfile', 'previewfile' ],
MIMETYPES.IMAGE.MIMETYPE_EXT,
{
thumbnailfile: getResumableUploadPath(),
previewfile: getResumableUploadPath()
}
)
const reqVideoFileUpdate = createReqFiles(
[ 'thumbnailfile', 'previewfile' ],
MIMETYPES.IMAGE.MIMETYPE_EXT,
@ -111,18 +126,39 @@ videosRouter.get('/',
commonVideosFiltersValidator,
asyncMiddleware(listVideos)
)
videosRouter.post('/upload',
authenticate,
reqVideoFileAdd,
asyncMiddleware(videosAddLegacyValidator),
asyncRetryTransactionMiddleware(addVideoLegacy)
)
videosRouter.post('/upload-resumable',
authenticate,
reqVideoFileAddResumable,
asyncMiddleware(videosAddResumableInitValidator),
uploadxMiddleware
)
videosRouter.delete('/upload-resumable',
authenticate,
uploadxMiddleware
)
videosRouter.put('/upload-resumable',
authenticate,
uploadxMiddleware, // uploadx doesn't use call next() before the file upload completes
asyncMiddleware(videosAddResumableValidator),
asyncMiddleware(addVideoResumable)
)
videosRouter.put('/:id',
authenticate,
reqVideoFileUpdate,
asyncMiddleware(videosUpdateValidator),
asyncRetryTransactionMiddleware(updateVideo)
)
videosRouter.post('/upload',
authenticate,
reqVideoFileAdd,
asyncMiddleware(videosAddValidator),
asyncRetryTransactionMiddleware(addVideo)
)
videosRouter.get('/:id/description',
asyncMiddleware(videosGetValidator),
@ -157,23 +193,23 @@ export {
// ---------------------------------------------------------------------------
function listVideoCategories (req: express.Request, res: express.Response) {
function listVideoCategories (_req: express.Request, res: express.Response) {
res.json(VIDEO_CATEGORIES)
}
function listVideoLicences (req: express.Request, res: express.Response) {
function listVideoLicences (_req: express.Request, res: express.Response) {
res.json(VIDEO_LICENCES)
}
function listVideoLanguages (req: express.Request, res: express.Response) {
function listVideoLanguages (_req: express.Request, res: express.Response) {
res.json(VIDEO_LANGUAGES)
}
function listVideoPrivacies (req: express.Request, res: express.Response) {
function listVideoPrivacies (_req: express.Request, res: express.Response) {
res.json(VIDEO_PRIVACIES)
}
async function addVideo (req: express.Request, res: express.Response) {
async function addVideoLegacy (req: express.Request, res: express.Response) {
// Uploading the video could be long
// Set timeout to 10 minutes, as Express's default is 2 minutes
req.setTimeout(1000 * 60 * 10, () => {
@ -183,13 +219,42 @@ async function addVideo (req: express.Request, res: express.Response) {
const videoPhysicalFile = req.files['videofile'][0]
const videoInfo: VideoCreate = req.body
const files = req.files
const videoData = buildLocalVideoFromReq(videoInfo, res.locals.videoChannel.id)
videoData.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED
videoData.duration = videoPhysicalFile['duration'] // duration was added by a previous middleware
return addVideo({ res, videoPhysicalFile, videoInfo, files })
}
async function addVideoResumable (_req: express.Request, res: express.Response) {
const videoPhysicalFile = res.locals.videoFileResumable
const videoInfo = videoPhysicalFile.metadata
const files = { previewfile: videoInfo.previewfile }
// Don't need the meta file anymore
await deleteResumableUploadMetaFile(videoPhysicalFile.path)
return addVideo({ res, videoPhysicalFile, videoInfo, files })
}
async function addVideo (options: {
res: express.Response
videoPhysicalFile: express.VideoUploadFile
videoInfo: VideoCreate
files: express.UploadFiles
}) {
const { res, videoPhysicalFile, videoInfo, files } = options
const videoChannel = res.locals.videoChannel
const user = res.locals.oauth.token.User
const videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id)
videoData.state = CONFIG.TRANSCODING.ENABLED
? VideoState.TO_TRANSCODE
: VideoState.PUBLISHED
videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware
const video = new VideoModel(videoData) as MVideoFullLight
video.VideoChannel = res.locals.videoChannel
video.VideoChannel = videoChannel
video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object
const videoFile = new VideoFileModel({
@ -217,7 +282,7 @@ async function addVideo (req: express.Request, res: express.Response) {
const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({
video,
files: req.files,
files,
fallback: type => generateVideoMiniature({ video, videoFile, type })
})
@ -253,7 +318,7 @@ async function addVideo (req: express.Request, res: express.Response) {
await autoBlacklistVideoIfNeeded({
video,
user: res.locals.oauth.token.User,
user,
isRemote: false,
isNew: true,
transaction: t
@ -282,7 +347,7 @@ async function addVideo (req: express.Request, res: express.Response) {
.catch(err => logger.error('Cannot federate or notify video creation %s', video.url, { err, ...lTags(video.uuid) }))
if (video.state === VideoState.TO_TRANSCODE) {
await addOptimizeOrMergeAudioJob(videoCreated, videoFile, res.locals.oauth.token.User)
await addOptimizeOrMergeAudioJob(videoCreated, videoFile, user)
}
Hooks.runAction('action:api.video.uploaded', { video: videoCreated })

View File

@ -1,6 +1,7 @@
import 'multer'
import validator from 'validator'
import { UploadFilesForCheck } from 'express'
import { sep } from 'path'
import validator from 'validator'
function exists (value: any) {
return value !== undefined && value !== null
@ -108,7 +109,7 @@ function isFileFieldValid (
}
function isFileMimeTypeValid (
files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[],
files: UploadFilesForCheck,
mimeTypeRegex: string,
field: string,
optional = false

View File

@ -1,4 +1,6 @@
import { UploadFilesForCheck } from 'express'
import { values } from 'lodash'
import * as magnetUtil from 'magnet-uri'
import validator from 'validator'
import { VideoFilter, VideoPrivacy, VideoRateType } from '../../../shared'
import {
@ -6,13 +8,12 @@ import {
MIMETYPES,
VIDEO_CATEGORIES,
VIDEO_LICENCES,
VIDEO_LIVE,
VIDEO_PRIVACIES,
VIDEO_RATE_TYPES,
VIDEO_STATES,
VIDEO_LIVE
VIDEO_STATES
} from '../../initializers/constants'
import { exists, isArray, isDateValid, isFileMimeTypeValid, isFileValid } from './misc'
import * as magnetUtil from 'magnet-uri'
const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS
@ -81,7 +82,7 @@ function isVideoFileExtnameValid (value: string) {
return exists(value) && (value === VIDEO_LIVE.EXTENSION || MIMETYPES.VIDEO.EXT_MIMETYPE[value] !== undefined)
}
function isVideoFileMimeTypeValid (files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[]) {
function isVideoFileMimeTypeValid (files: UploadFilesForCheck) {
return isFileMimeTypeValid(files, MIMETYPES.VIDEO.MIMETYPES_REGEX, 'videofile')
}

View File

@ -2,7 +2,7 @@ import * as express from 'express'
import * as multer from 'multer'
import { REMOTE_SCHEME } from '../initializers/constants'
import { logger } from './logger'
import { deleteFileAsync, generateRandomString } from './utils'
import { deleteFileAndCatch, generateRandomString } from './utils'
import { extname } from 'path'
import { isArray } from './custom-validators/misc'
import { CONFIG } from '../initializers/config'
@ -36,15 +36,15 @@ function cleanUpReqFiles (req: { files: { [fieldname: string]: Express.Multer.Fi
if (!files) return
if (isArray(files)) {
(files as Express.Multer.File[]).forEach(f => deleteFileAsync(f.path))
(files as Express.Multer.File[]).forEach(f => deleteFileAndCatch(f.path))
return
}
for (const key of Object.keys(files)) {
const file = files[key]
if (isArray(file)) file.forEach(f => deleteFileAsync(f.path))
else deleteFileAsync(file.path)
if (isArray(file)) file.forEach(f => deleteFileAndCatch(f.path))
else deleteFileAndCatch(file.path)
}
}

21
server/helpers/upload.ts Normal file
View File

@ -0,0 +1,21 @@
import { METAFILE_EXTNAME } from '@uploadx/core'
import { remove } from 'fs-extra'
import { join } from 'path'
import { RESUMABLE_UPLOAD_DIRECTORY } from '../initializers/constants'
function getResumableUploadPath (filename?: string) {
if (filename) return join(RESUMABLE_UPLOAD_DIRECTORY, filename)
return RESUMABLE_UPLOAD_DIRECTORY
}
function deleteResumableUploadMetaFile (filepath: string) {
return remove(filepath + METAFILE_EXTNAME)
}
// ---------------------------------------------------------------------------
export {
getResumableUploadPath,
deleteResumableUploadMetaFile
}

View File

@ -6,7 +6,7 @@ import { CONFIG } from '../initializers/config'
import { execPromise, execPromise2, randomBytesPromise, sha256 } from './core-utils'
import { logger } from './logger'
function deleteFileAsync (path: string) {
function deleteFileAndCatch (path: string) {
remove(path)
.catch(err => logger.error('Cannot delete the file %s asynchronously.', path, { err }))
}
@ -83,7 +83,7 @@ function getUUIDFromFilename (filename: string) {
// ---------------------------------------------------------------------------
export {
deleteFileAsync,
deleteFileAndCatch,
generateRandomString,
getFormattedObjects,
getSecureTorrentName,

View File

@ -208,7 +208,8 @@ const SCHEDULER_INTERVALS_MS = {
autoFollowIndexInstances: 60000 * 60 * 24, // 1 day
removeOldViews: 60000 * 60 * 24, // 1 day
removeOldHistory: 60000 * 60 * 24, // 1 day
updateInboxStats: 1000 * 60// 1 minute
updateInboxStats: 1000 * 60, // 1 minute
removeDanglingResumableUploads: 60000 * 60 * 16 // 16 hours
}
// ---------------------------------------------------------------------------
@ -285,6 +286,7 @@ const CONSTRAINTS_FIELDS = {
LIKES: { min: 0 },
DISLIKES: { min: 0 },
FILE_SIZE: { min: -1 },
PARTIAL_UPLOAD_SIZE: { max: 50 * 1024 * 1024 * 1024 }, // 50GB
URL: { min: 3, max: 2000 } // Length
},
VIDEO_PLAYLISTS: {
@ -645,6 +647,7 @@ const LRU_CACHE = {
}
}
const RESUMABLE_UPLOAD_DIRECTORY = join(CONFIG.STORAGE.TMP_DIR, 'resumable-uploads')
const HLS_STREAMING_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls')
const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls')
@ -819,6 +822,7 @@ export {
PEERTUBE_VERSION,
LAZY_STATIC_PATHS,
SEARCH_INDEX,
RESUMABLE_UPLOAD_DIRECTORY,
HLS_REDUNDANCY_DIRECTORY,
P2P_MEDIA_LOADER_PEER_VERSION,
ACTOR_IMAGES_SIZE,

View File

@ -6,7 +6,7 @@ import { UserModel } from '../models/account/user'
import { ApplicationModel } from '../models/application/application'
import { OAuthClientModel } from '../models/oauth/oauth-client'
import { applicationExist, clientsExist, usersExist } from './checker-after-init'
import { FILES_CACHE, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION } from './constants'
import { FILES_CACHE, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION, RESUMABLE_UPLOAD_DIRECTORY } from './constants'
import { sequelizeTypescript } from './database'
import { ensureDir, remove } from 'fs-extra'
import { CONFIG } from './config'
@ -79,6 +79,9 @@ function createDirectoriesIfNotExist () {
// Playlist directories
tasks.push(ensureDir(HLS_STREAMING_PLAYLIST_DIRECTORY))
// Resumable upload directory
tasks.push(ensureDir(RESUMABLE_UPLOAD_DIRECTORY))
return Promise.all(tasks)
}

View File

@ -1,6 +1,8 @@
import { VideoUploadFile } from 'express'
import { PathLike } from 'fs-extra'
import { Transaction } from 'sequelize/types'
import { AbuseAuditView, auditLoggerFactory } from '@server/helpers/audit-logger'
import { afterCommitIfTransaction } from '@server/helpers/database-utils'
import { logger } from '@server/helpers/logger'
import { AbuseModel } from '@server/models/abuse/abuse'
import { VideoAbuseModel } from '@server/models/abuse/video-abuse'
@ -28,7 +30,6 @@ import { VideoModel } from '../models/video/video'
import { VideoCommentModel } from '../models/video/video-comment'
import { sendAbuse } from './activitypub/send/send-flag'
import { Notifier } from './notifier'
import { afterCommitIfTransaction } from '@server/helpers/database-utils'
export type AcceptResult = {
accepted: boolean
@ -38,7 +39,7 @@ export type AcceptResult = {
// Can be filtered by plugins
function isLocalVideoAccepted (object: {
videoBody: VideoCreate
videoFile: Express.Multer.File & { duration?: number }
videoFile: VideoUploadFile
user: UserModel
}): AcceptResult {
return { accepted: true }

View File

@ -0,0 +1,61 @@
import * as bluebird from 'bluebird'
import { readdir, remove, stat } from 'fs-extra'
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { getResumableUploadPath } from '@server/helpers/upload'
import { SCHEDULER_INTERVALS_MS } from '@server/initializers/constants'
import { METAFILE_EXTNAME } from '@uploadx/core'
import { AbstractScheduler } from './abstract-scheduler'
const lTags = loggerTagsFactory('scheduler', 'resumable-upload', 'cleaner')
export class RemoveDanglingResumableUploadsScheduler extends AbstractScheduler {
private static instance: AbstractScheduler
private lastExecutionTimeMs: number
protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.removeDanglingResumableUploads
private constructor () {
super()
this.lastExecutionTimeMs = new Date().getTime()
}
protected async internalExecute () {
const path = getResumableUploadPath()
const files = await readdir(path)
const metafiles = files.filter(f => f.endsWith(METAFILE_EXTNAME))
if (metafiles.length === 0) return
logger.debug('Reading resumable video upload folder %s with %d files', path, metafiles.length, lTags())
try {
await bluebird.map(metafiles, metafile => {
return this.deleteIfOlderThan(metafile, this.lastExecutionTimeMs)
}, { concurrency: 5 })
} catch (error) {
logger.error('Failed to handle file during resumable video upload folder cleanup', { error, ...lTags() })
} finally {
this.lastExecutionTimeMs = new Date().getTime()
}
}
private async deleteIfOlderThan (metafile: string, olderThan: number) {
const metafilePath = getResumableUploadPath(metafile)
const statResult = await stat(metafilePath)
// Delete uploads that started since a long time
if (statResult.ctimeMs < olderThan) {
await remove(metafilePath)
const datafile = metafilePath.replace(new RegExp(`${METAFILE_EXTNAME}$`), '')
await remove(datafile)
}
}
static get Instance () {
return this.instance || (this.instance = new this())
}
}

View File

@ -1,3 +1,4 @@
import { UploadFiles } from 'express'
import { Transaction } from 'sequelize/types'
import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY } from '@server/initializers/constants'
import { sequelizeTypescript } from '@server/initializers/database'
@ -32,7 +33,7 @@ function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): Fil
async function buildVideoThumbnailsFromReq (options: {
video: MVideoThumbnail
files: { [fieldname: string]: Express.Multer.File[] } | Express.Multer.File[]
files: UploadFiles
fallback: (type: ThumbnailType) => Promise<MThumbnail>
automaticallyGenerated?: boolean
}) {

View File

@ -3,6 +3,7 @@ import { NextFunction, Request, RequestHandler, Response } from 'express'
import { ValidationChain } from 'express-validator'
import { ExpressPromiseHandler } from '@server/types/express'
import { retryTransactionWrapper } from '../helpers/database-utils'
import { HttpMethod, HttpStatusCode } from '@shared/core-utils'
// Syntactic sugar to avoid try/catch in express controllers
// Thanks: https://medium.com/@Abazhenov/using-async-await-in-express-with-node-8-b8af872c0016

View File

@ -1,9 +1,10 @@
import * as express from 'express'
import { body, param, query, ValidationChain } from 'express-validator'
import { body, header, param, query, ValidationChain } from 'express-validator'
import { getResumableUploadPath } from '@server/helpers/upload'
import { isAbleToUploadVideo } from '@server/lib/user'
import { getServerActor } from '@server/models/application/application'
import { ExpressPromiseHandler } from '@server/types/express'
import { MVideoWithRights } from '@server/types/models'
import { MUserAccountId, MVideoWithRights } from '@server/types/models'
import { ServerErrorCode, UserRight, VideoChangeOwnershipStatus, VideoPrivacy } from '../../../../shared'
import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
import { VideoChangeOwnershipAccept } from '../../../../shared/models/videos/video-change-ownership-accept.model'
@ -47,6 +48,7 @@ import {
doesVideoExist,
doesVideoFileOfVideoExist
} from '../../../helpers/middlewares'
import { deleteFileAndCatch } from '../../../helpers/utils'
import { getVideoWithAttributes } from '../../../helpers/video'
import { CONFIG } from '../../../initializers/config'
import { CONSTRAINTS_FIELDS, OVERVIEWS } from '../../../initializers/constants'
@ -57,7 +59,7 @@ import { VideoModel } from '../../../models/video/video'
import { authenticatePromiseIfNeeded } from '../../auth'
import { areValidationErrors } from '../utils'
const videosAddValidator = getCommonVideoEditAttributes().concat([
const videosAddLegacyValidator = getCommonVideoEditAttributes().concat([
body('videofile')
.custom((value, { req }) => isFileFieldValid(req.files, 'videofile'))
.withMessage('Should have a file'),
@ -73,59 +75,122 @@ const videosAddValidator = getCommonVideoEditAttributes().concat([
logger.debug('Checking videosAdd parameters', { parameters: req.body, files: req.files })
if (areValidationErrors(req, res)) return cleanUpReqFiles(req)
if (areErrorsInScheduleUpdate(req, res)) return cleanUpReqFiles(req)
const videoFile: Express.Multer.File & { duration?: number } = req.files['videofile'][0]
const videoFile: express.VideoUploadFile = req.files['videofile'][0]
const user = res.locals.oauth.token.User
if (!await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req)
if (!isVideoFileMimeTypeValid(req.files)) {
res.status(HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415)
.json({
error: 'This file is not supported. Please, make sure it is of the following type: ' +
CONSTRAINTS_FIELDS.VIDEOS.EXTNAME.join(', ')
})
if (!await commonVideoChecksPass({ req, res, user, videoFileSize: videoFile.size, files: req.files })) {
return cleanUpReqFiles(req)
}
if (!isVideoFileSizeValid(videoFile.size.toString())) {
res.status(HttpStatusCode.PAYLOAD_TOO_LARGE_413)
.json({
error: 'This file is too large.'
})
return cleanUpReqFiles(req)
}
if (await isAbleToUploadVideo(user.id, videoFile.size) === false) {
res.status(HttpStatusCode.PAYLOAD_TOO_LARGE_413)
.json({ error: 'The user video quota is exceeded with this video.' })
return cleanUpReqFiles(req)
}
let duration: number
try {
duration = await getDurationFromVideoFile(videoFile.path)
if (!videoFile.duration) await addDurationToVideo(videoFile)
} catch (err) {
logger.error('Invalid input file in videosAddValidator.', { err })
logger.error('Invalid input file in videosAddLegacyValidator.', { err })
res.status(HttpStatusCode.UNPROCESSABLE_ENTITY_422)
.json({ error: 'Video file unreadable.' })
return cleanUpReqFiles(req)
}
videoFile.duration = duration
if (!await isVideoAccepted(req, res, videoFile)) return cleanUpReqFiles(req)
return next()
}
])
/**
* Gets called after the last PUT request
*/
const videosAddResumableValidator = [
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
const user = res.locals.oauth.token.User
const body: express.CustomUploadXFile<express.UploadXFileMetadata> = req.body
const file = { ...body, duration: undefined, path: getResumableUploadPath(body.id), filename: body.metadata.filename }
const cleanup = () => deleteFileAndCatch(file.path)
if (!await doesVideoChannelOfAccountExist(file.metadata.channelId, user, res)) return cleanup()
try {
if (!file.duration) await addDurationToVideo(file)
} catch (err) {
logger.error('Invalid input file in videosAddResumableValidator.', { err })
res.status(HttpStatusCode.UNPROCESSABLE_ENTITY_422)
.json({ error: 'Video file unreadable.' })
return cleanup()
}
if (!await isVideoAccepted(req, res, file)) return cleanup()
res.locals.videoFileResumable = file
return next()
}
]
/**
* File is created in POST initialisation, and its body is saved as a 'metadata' field is saved by uploadx for later use.
* see https://github.com/kukhariev/node-uploadx/blob/dc9fb4a8ac5a6f481902588e93062f591ec6ef03/packages/core/src/handlers/uploadx.ts
*
* Uploadx doesn't use next() until the upload completes, so this middleware has to be placed before uploadx
* see https://github.com/kukhariev/node-uploadx/blob/dc9fb4a8ac5a6f481902588e93062f591ec6ef03/packages/core/src/handlers/base-handler.ts
*
*/
const videosAddResumableInitValidator = getCommonVideoEditAttributes().concat([
body('filename')
.isString()
.exists()
.withMessage('Should have a valid filename'),
body('name')
.trim()
.custom(isVideoNameValid)
.withMessage('Should have a valid name'),
body('channelId')
.customSanitizer(toIntOrNull)
.custom(isIdValid).withMessage('Should have correct video channel id'),
header('x-upload-content-length')
.isNumeric()
.exists()
.withMessage('Should specify the file length'),
header('x-upload-content-type')
.isString()
.exists()
.withMessage('Should specify the file mimetype'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
const videoFileMetadata = {
mimetype: req.headers['x-upload-content-type'] as string,
size: +req.headers['x-upload-content-length'],
originalname: req.body.name
}
const user = res.locals.oauth.token.User
const cleanup = () => cleanUpReqFiles(req)
logger.debug('Checking videosAddResumableInitValidator parameters and headers', {
parameters: req.body,
headers: req.headers,
files: req.files
})
if (areValidationErrors(req, res)) return cleanup()
const files = { videofile: [ videoFileMetadata ] }
if (!await commonVideoChecksPass({ req, res, user, videoFileSize: videoFileMetadata.size, files })) return cleanup()
// multer required unsetting the Content-Type, now we can set it for node-uploadx
req.headers['content-type'] = 'application/json; charset=utf-8'
// place previewfile in metadata so that uploadx saves it in .META
if (req.files['previewfile']) req.body.previewfile = req.files['previewfile']
return next()
}
])
const videosUpdateValidator = getCommonVideoEditAttributes().concat([
param('id').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid id'),
body('name')
@ -478,7 +543,10 @@ const commonVideosFiltersValidator = [
// ---------------------------------------------------------------------------
export {
videosAddValidator,
videosAddLegacyValidator,
videosAddResumableValidator,
videosAddResumableInitValidator,
videosUpdateValidator,
videosGetValidator,
videoFileMetadataGetValidator,
@ -515,7 +583,51 @@ function areErrorsInScheduleUpdate (req: express.Request, res: express.Response)
return false
}
async function isVideoAccepted (req: express.Request, res: express.Response, videoFile: Express.Multer.File & { duration?: number }) {
async function commonVideoChecksPass (parameters: {
req: express.Request
res: express.Response
user: MUserAccountId
videoFileSize: number
files: express.UploadFilesForCheck
}): Promise<boolean> {
const { req, res, user, videoFileSize, files } = parameters
if (areErrorsInScheduleUpdate(req, res)) return false
if (!await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return false
if (!isVideoFileMimeTypeValid(files)) {
res.status(HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415)
.json({
error: 'This file is not supported. Please, make sure it is of the following type: ' +
CONSTRAINTS_FIELDS.VIDEOS.EXTNAME.join(', ')
})
return false
}
if (!isVideoFileSizeValid(videoFileSize.toString())) {
res.status(HttpStatusCode.PAYLOAD_TOO_LARGE_413)
.json({ error: 'This file is too large. It exceeds the maximum file size authorized.' })
return false
}
if (await isAbleToUploadVideo(user.id, videoFileSize) === false) {
res.status(HttpStatusCode.PAYLOAD_TOO_LARGE_413)
.json({ error: 'The user video quota is exceeded with this video.' })
return false
}
return true
}
export async function isVideoAccepted (
req: express.Request,
res: express.Response,
videoFile: express.VideoUploadFile
) {
// Check we accept this video
const acceptParameters = {
videoBody: req.body,
@ -538,3 +650,11 @@ async function isVideoAccepted (req: express.Request, res: express.Response, vid
return true
}
async function addDurationToVideo (videoFile: { path: string, duration?: number }) {
const duration: number = await getDurationFromVideoFile(videoFile.path)
if (isNaN(duration)) throw new Error(`Couldn't get video duration`)
videoFile.duration = duration
}

View File

@ -13,6 +13,7 @@ import './plugins'
import './redundancy'
import './search'
import './services'
import './upload-quota'
import './user-notifications'
import './user-subscriptions'
import './users'

View File

@ -0,0 +1,152 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import 'mocha'
import { expect } from 'chai'
import { HttpStatusCode, randomInt } from '@shared/core-utils'
import { getGoodVideoUrl, getMagnetURI, getMyVideoImports, importVideo } from '@shared/extra-utils/videos/video-imports'
import { MyUser, VideoImport, VideoImportState, VideoPrivacy } from '@shared/models'
import {
cleanupTests,
flushAndRunServer,
getMyUserInformation,
immutableAssign,
registerUser,
ServerInfo,
setAccessTokensToServers,
setDefaultVideoChannel,
updateUser,
uploadVideo,
userLogin,
waitJobs
} from '../../../../shared/extra-utils'
describe('Test upload quota', function () {
let server: ServerInfo
let rootId: number
// ---------------------------------------------------------------
before(async function () {
this.timeout(30000)
server = await flushAndRunServer(1)
await setAccessTokensToServers([ server ])
await setDefaultVideoChannel([ server ])
const res = await getMyUserInformation(server.url, server.accessToken)
rootId = (res.body as MyUser).id
await updateUser({
url: server.url,
userId: rootId,
accessToken: server.accessToken,
videoQuota: 42
})
})
describe('When having a video quota', function () {
it('Should fail with a registered user having too many videos with legacy upload', async function () {
this.timeout(30000)
const user = { username: 'registered' + randomInt(1, 1500), password: 'password' }
await registerUser(server.url, user.username, user.password)
const userAccessToken = await userLogin(server, user)
const videoAttributes = { fixture: 'video_short2.webm' }
for (let i = 0; i < 5; i++) {
await uploadVideo(server.url, userAccessToken, videoAttributes)
}
await uploadVideo(server.url, userAccessToken, videoAttributes, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'legacy')
})
it('Should fail with a registered user having too many videos with resumable upload', async function () {
this.timeout(30000)
const user = { username: 'registered' + randomInt(1, 1500), password: 'password' }
await registerUser(server.url, user.username, user.password)
const userAccessToken = await userLogin(server, user)
const videoAttributes = { fixture: 'video_short2.webm' }
for (let i = 0; i < 5; i++) {
await uploadVideo(server.url, userAccessToken, videoAttributes)
}
await uploadVideo(server.url, userAccessToken, videoAttributes, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'resumable')
})
it('Should fail to import with HTTP/Torrent/magnet', async function () {
this.timeout(120000)
const baseAttributes = {
channelId: server.videoChannel.id,
privacy: VideoPrivacy.PUBLIC
}
await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { targetUrl: getGoodVideoUrl() }))
await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { magnetUri: getMagnetURI() }))
await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { torrentfile: 'video-720p.torrent' as any }))
await waitJobs([ server ])
const res = await getMyVideoImports(server.url, server.accessToken)
expect(res.body.total).to.equal(3)
const videoImports: VideoImport[] = res.body.data
expect(videoImports).to.have.lengthOf(3)
for (const videoImport of videoImports) {
expect(videoImport.state.id).to.equal(VideoImportState.FAILED)
expect(videoImport.error).not.to.be.undefined
expect(videoImport.error).to.contain('user video quota is exceeded')
}
})
})
describe('When having a daily video quota', function () {
it('Should fail with a user having too many videos daily', async function () {
await updateUser({
url: server.url,
userId: rootId,
accessToken: server.accessToken,
videoQuotaDaily: 42
})
await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'legacy')
await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'resumable')
})
})
describe('When having an absolute and daily video quota', function () {
it('Should fail if exceeding total quota', async function () {
await updateUser({
url: server.url,
userId: rootId,
accessToken: server.accessToken,
videoQuota: 42,
videoQuotaDaily: 1024 * 1024 * 1024
})
await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'legacy')
await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'resumable')
})
it('Should fail if exceeding daily quota', async function () {
await updateUser({
url: server.url,
userId: rootId,
accessToken: server.accessToken,
videoQuota: 1024 * 1024 * 1024,
videoQuotaDaily: 42
})
await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'legacy')
await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'resumable')
})
})
after(async function () {
await cleanupTests([ server ])
})
})

View File

@ -1,10 +1,10 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import 'mocha'
import { expect } from 'chai'
import { omit } from 'lodash'
import { join } from 'path'
import { User, UserRole, VideoImport, VideoImportState } from '../../../../shared'
import { User, UserRole } from '../../../../shared'
import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
import {
addVideoChannel,
blockUser,
@ -29,7 +29,6 @@ import {
ServerInfo,
setAccessTokensToServers,
unblockUser,
updateUser,
uploadVideo,
userLogin
} from '../../../../shared/extra-utils'
@ -39,11 +38,7 @@ import {
checkBadSortPagination,
checkBadStartPagination
} from '../../../../shared/extra-utils/requests/check-api-params'
import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
import { getGoodVideoUrl, getMagnetURI, getMyVideoImports, importVideo } from '../../../../shared/extra-utils/videos/video-imports'
import { UserAdminFlag } from '../../../../shared/models/users/user-flag.model'
import { VideoPrivacy } from '../../../../shared/models/videos'
import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
describe('Test users API validators', function () {
const path = '/api/v1/users/'
@ -1093,102 +1088,6 @@ describe('Test users API validators', function () {
})
})
describe('When having a video quota', function () {
it('Should fail with a user having too many videos', async function () {
await updateUser({
url: server.url,
userId: rootId,
accessToken: server.accessToken,
videoQuota: 42
})
await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413)
})
it('Should fail with a registered user having too many videos', async function () {
this.timeout(30000)
const user = {
username: 'user3',
password: 'my super password'
}
userAccessToken = await userLogin(server, user)
const videoAttributes = { fixture: 'video_short2.webm' }
await uploadVideo(server.url, userAccessToken, videoAttributes)
await uploadVideo(server.url, userAccessToken, videoAttributes)
await uploadVideo(server.url, userAccessToken, videoAttributes)
await uploadVideo(server.url, userAccessToken, videoAttributes)
await uploadVideo(server.url, userAccessToken, videoAttributes)
await uploadVideo(server.url, userAccessToken, videoAttributes, HttpStatusCode.PAYLOAD_TOO_LARGE_413)
})
it('Should fail to import with HTTP/Torrent/magnet', async function () {
this.timeout(120000)
const baseAttributes = {
channelId: 1,
privacy: VideoPrivacy.PUBLIC
}
await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { targetUrl: getGoodVideoUrl() }))
await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { magnetUri: getMagnetURI() }))
await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { torrentfile: 'video-720p.torrent' as any }))
await waitJobs([ server ])
const res = await getMyVideoImports(server.url, server.accessToken)
expect(res.body.total).to.equal(3)
const videoImports: VideoImport[] = res.body.data
expect(videoImports).to.have.lengthOf(3)
for (const videoImport of videoImports) {
expect(videoImport.state.id).to.equal(VideoImportState.FAILED)
expect(videoImport.error).not.to.be.undefined
expect(videoImport.error).to.contain('user video quota is exceeded')
}
})
})
describe('When having a daily video quota', function () {
it('Should fail with a user having too many videos daily', async function () {
await updateUser({
url: server.url,
userId: rootId,
accessToken: server.accessToken,
videoQuotaDaily: 42
})
await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413)
})
})
describe('When having an absolute and daily video quota', function () {
it('Should fail if exceeding total quota', async function () {
await updateUser({
url: server.url,
userId: rootId,
accessToken: server.accessToken,
videoQuota: 42,
videoQuotaDaily: 1024 * 1024 * 1024
})
await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413)
})
it('Should fail if exceeding daily quota', async function () {
await updateUser({
url: server.url,
userId: rootId,
accessToken: server.accessToken,
videoQuota: 1024 * 1024 * 1024,
videoQuotaDaily: 42
})
await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413)
})
})
describe('When asking a password reset', function () {
const path = '/api/v1/users/ask-reset-password'

View File

@ -1,11 +1,12 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import 'mocha'
import * as chai from 'chai'
import { omit } from 'lodash'
import 'mocha'
import { join } from 'path'
import { VideoPrivacy } from '../../../../shared/models/videos/video-privacy.enum'
import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
import {
checkUploadVideoParam,
cleanupTests,
createUser,
flushAndRunServer,
@ -18,17 +19,18 @@ import {
makePutBodyRequest,
makeUploadRequest,
removeVideo,
root,
ServerInfo,
setAccessTokensToServers,
userLogin,
root
userLogin
} from '../../../../shared/extra-utils'
import {
checkBadCountPagination,
checkBadSortPagination,
checkBadStartPagination
} from '../../../../shared/extra-utils/requests/check-api-params'
import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
import { VideoPrivacy } from '../../../../shared/models/videos/video-privacy.enum'
import { randomInt } from '@shared/core-utils'
const expect = chai.expect
@ -183,7 +185,7 @@ describe('Test videos API validator', function () {
describe('When adding a video', function () {
let baseCorrectParams
const baseCorrectAttaches = {
videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.webm')
fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.webm')
}
before(function () {
@ -206,256 +208,243 @@ describe('Test videos API validator', function () {
}
})
it('Should fail with nothing', async function () {
const fields = {}
const attaches = {}
await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches })
})
function runSuite (mode: 'legacy' | 'resumable') {
it('Should fail without name', async function () {
const fields = omit(baseCorrectParams, 'name')
const attaches = baseCorrectAttaches
await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches })
})
it('Should fail with a long name', async function () {
const fields = immutableAssign(baseCorrectParams, { name: 'super'.repeat(65) })
const attaches = baseCorrectAttaches
await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches })
})
it('Should fail with a bad category', async function () {
const fields = immutableAssign(baseCorrectParams, { category: 125 })
const attaches = baseCorrectAttaches
await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches })
})
it('Should fail with a bad licence', async function () {
const fields = immutableAssign(baseCorrectParams, { licence: 125 })
const attaches = baseCorrectAttaches
await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches })
})
it('Should fail with a bad language', async function () {
const fields = immutableAssign(baseCorrectParams, { language: 'a'.repeat(15) })
const attaches = baseCorrectAttaches
await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches })
})
it('Should fail with a long description', async function () {
const fields = immutableAssign(baseCorrectParams, { description: 'super'.repeat(2500) })
const attaches = baseCorrectAttaches
await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches })
})
it('Should fail with a long support text', async function () {
const fields = immutableAssign(baseCorrectParams, { support: 'super'.repeat(201) })
const attaches = baseCorrectAttaches
await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches })
})
it('Should fail without a channel', async function () {
const fields = omit(baseCorrectParams, 'channelId')
const attaches = baseCorrectAttaches
await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches })
})
it('Should fail with a bad channel', async function () {
const fields = immutableAssign(baseCorrectParams, { channelId: 545454 })
const attaches = baseCorrectAttaches
await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches })
})
it('Should fail with another user channel', async function () {
const user = {
username: 'fake',
password: 'fake_password'
}
await createUser({ url: server.url, accessToken: server.accessToken, username: user.username, password: user.password })
const accessTokenUser = await userLogin(server, user)
const res = await getMyUserInformation(server.url, accessTokenUser)
const customChannelId = res.body.videoChannels[0].id
const fields = immutableAssign(baseCorrectParams, { channelId: customChannelId })
const attaches = baseCorrectAttaches
await makeUploadRequest({ url: server.url, path: path + '/upload', token: userAccessToken, fields, attaches })
})
it('Should fail with too many tags', async function () {
const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6' ] })
const attaches = baseCorrectAttaches
await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches })
})
it('Should fail with a tag length too low', async function () {
const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 't' ] })
const attaches = baseCorrectAttaches
await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches })
})
it('Should fail with a tag length too big', async function () {
const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'my_super_tag_too_long_long_long_long_long_long' ] })
const attaches = baseCorrectAttaches
await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches })
})
it('Should fail with a bad schedule update (miss updateAt)', async function () {
const fields = immutableAssign(baseCorrectParams, { 'scheduleUpdate[privacy]': VideoPrivacy.PUBLIC })
const attaches = baseCorrectAttaches
await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches })
})
it('Should fail with a bad schedule update (wrong updateAt)', async function () {
const fields = immutableAssign(baseCorrectParams, {
'scheduleUpdate[privacy]': VideoPrivacy.PUBLIC,
'scheduleUpdate[updateAt]': 'toto'
})
const attaches = baseCorrectAttaches
await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches })
})
it('Should fail with a bad originally published at attribute', async function () {
const fields = immutableAssign(baseCorrectParams, { originallyPublishedAt: 'toto' })
const attaches = baseCorrectAttaches
await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches })
})
it('Should fail without an input file', async function () {
const fields = baseCorrectParams
const attaches = {}
await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches })
})
it('Should fail with an incorrect input file', async function () {
const fields = baseCorrectParams
let attaches = {
videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short_fake.webm')
}
await makeUploadRequest({
url: server.url,
path: path + '/upload',
token: server.accessToken,
fields,
attaches,
statusCodeExpected: HttpStatusCode.UNPROCESSABLE_ENTITY_422
it('Should fail with nothing', async function () {
const fields = {}
const attaches = {}
await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode)
})
attaches = {
videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mkv')
}
await makeUploadRequest({
url: server.url,
path: path + '/upload',
token: server.accessToken,
fields,
attaches,
statusCodeExpected: HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415
})
})
it('Should fail with an incorrect thumbnail file', async function () {
const fields = baseCorrectParams
const attaches = {
thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4'),
videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4')
}
await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches })
})
it('Should fail with a big thumbnail file', async function () {
const fields = baseCorrectParams
const attaches = {
thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'preview-big.png'),
videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4')
}
await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches })
})
it('Should fail with an incorrect preview file', async function () {
const fields = baseCorrectParams
const attaches = {
previewfile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4'),
videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4')
}
await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches })
})
it('Should fail with a big preview file', async function () {
const fields = baseCorrectParams
const attaches = {
previewfile: join(root(), 'server', 'tests', 'fixtures', 'preview-big.png'),
videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4')
}
await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches })
})
it('Should succeed with the correct parameters', async function () {
this.timeout(10000)
const fields = baseCorrectParams
{
it('Should fail without name', async function () {
const fields = omit(baseCorrectParams, 'name')
const attaches = baseCorrectAttaches
await makeUploadRequest({
url: server.url,
path: path + '/upload',
token: server.accessToken,
fields,
attaches,
statusCodeExpected: HttpStatusCode.OK_200
})
}
{
const attaches = immutableAssign(baseCorrectAttaches, {
videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4')
})
await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode)
})
await makeUploadRequest({
url: server.url,
path: path + '/upload',
token: server.accessToken,
fields,
attaches,
statusCodeExpected: HttpStatusCode.OK_200
})
}
it('Should fail with a long name', async function () {
const fields = immutableAssign(baseCorrectParams, { name: 'super'.repeat(65) })
const attaches = baseCorrectAttaches
{
const attaches = immutableAssign(baseCorrectAttaches, {
videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.ogv')
})
await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode)
})
await makeUploadRequest({
url: server.url,
path: path + '/upload',
token: server.accessToken,
fields,
attaches,
statusCodeExpected: HttpStatusCode.OK_200
it('Should fail with a bad category', async function () {
const fields = immutableAssign(baseCorrectParams, { category: 125 })
const attaches = baseCorrectAttaches
await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode)
})
it('Should fail with a bad licence', async function () {
const fields = immutableAssign(baseCorrectParams, { licence: 125 })
const attaches = baseCorrectAttaches
await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode)
})
it('Should fail with a bad language', async function () {
const fields = immutableAssign(baseCorrectParams, { language: 'a'.repeat(15) })
const attaches = baseCorrectAttaches
await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode)
})
it('Should fail with a long description', async function () {
const fields = immutableAssign(baseCorrectParams, { description: 'super'.repeat(2500) })
const attaches = baseCorrectAttaches
await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode)
})
it('Should fail with a long support text', async function () {
const fields = immutableAssign(baseCorrectParams, { support: 'super'.repeat(201) })
const attaches = baseCorrectAttaches
await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode)
})
it('Should fail without a channel', async function () {
const fields = omit(baseCorrectParams, 'channelId')
const attaches = baseCorrectAttaches
await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode)
})
it('Should fail with a bad channel', async function () {
const fields = immutableAssign(baseCorrectParams, { channelId: 545454 })
const attaches = baseCorrectAttaches
await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode)
})
it('Should fail with another user channel', async function () {
const user = {
username: 'fake' + randomInt(0, 1500),
password: 'fake_password'
}
await createUser({ url: server.url, accessToken: server.accessToken, username: user.username, password: user.password })
const accessTokenUser = await userLogin(server, user)
const res = await getMyUserInformation(server.url, accessTokenUser)
const customChannelId = res.body.videoChannels[0].id
const fields = immutableAssign(baseCorrectParams, { channelId: customChannelId })
const attaches = baseCorrectAttaches
await checkUploadVideoParam(server.url, userAccessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode)
})
it('Should fail with too many tags', async function () {
const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6' ] })
const attaches = baseCorrectAttaches
await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode)
})
it('Should fail with a tag length too low', async function () {
const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 't' ] })
const attaches = baseCorrectAttaches
await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode)
})
it('Should fail with a tag length too big', async function () {
const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'my_super_tag_too_long_long_long_long_long_long' ] })
const attaches = baseCorrectAttaches
await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode)
})
it('Should fail with a bad schedule update (miss updateAt)', async function () {
const fields = immutableAssign(baseCorrectParams, { scheduleUpdate: { privacy: VideoPrivacy.PUBLIC } })
const attaches = baseCorrectAttaches
await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode)
})
it('Should fail with a bad schedule update (wrong updateAt)', async function () {
const fields = immutableAssign(baseCorrectParams, {
scheduleUpdate: {
privacy: VideoPrivacy.PUBLIC,
updateAt: 'toto'
}
})
}
const attaches = baseCorrectAttaches
await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode)
})
it('Should fail with a bad originally published at attribute', async function () {
const fields = immutableAssign(baseCorrectParams, { originallyPublishedAt: 'toto' })
const attaches = baseCorrectAttaches
await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode)
})
it('Should fail without an input file', async function () {
const fields = baseCorrectParams
const attaches = {}
await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode)
})
it('Should fail with an incorrect input file', async function () {
const fields = baseCorrectParams
let attaches = { fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short_fake.webm') }
await checkUploadVideoParam(
server.url,
server.accessToken,
{ ...fields, ...attaches },
HttpStatusCode.UNPROCESSABLE_ENTITY_422,
mode
)
attaches = { fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mkv') }
await checkUploadVideoParam(
server.url,
server.accessToken,
{ ...fields, ...attaches },
HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415,
mode
)
})
it('Should fail with an incorrect thumbnail file', async function () {
const fields = baseCorrectParams
const attaches = {
thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4'),
fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4')
}
await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode)
})
it('Should fail with a big thumbnail file', async function () {
const fields = baseCorrectParams
const attaches = {
thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'preview-big.png'),
fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4')
}
await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode)
})
it('Should fail with an incorrect preview file', async function () {
const fields = baseCorrectParams
const attaches = {
previewfile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4'),
fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4')
}
await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode)
})
it('Should fail with a big preview file', async function () {
const fields = baseCorrectParams
const attaches = {
previewfile: join(root(), 'server', 'tests', 'fixtures', 'preview-big.png'),
fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4')
}
await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode)
})
it('Should succeed with the correct parameters', async function () {
this.timeout(10000)
const fields = baseCorrectParams
{
const attaches = baseCorrectAttaches
await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.OK_200, mode)
}
{
const attaches = immutableAssign(baseCorrectAttaches, {
videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4')
})
await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.OK_200, mode)
}
{
const attaches = immutableAssign(baseCorrectAttaches, {
videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.ogv')
})
await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.OK_200, mode)
}
})
}
describe('Resumable upload', function () {
runSuite('resumable')
})
describe('Legacy upload', function () {
runSuite('legacy')
})
})
@ -678,7 +667,7 @@ describe('Test videos API validator', function () {
})
expect(res.body.data).to.be.an('array')
expect(res.body.data.length).to.equal(3)
expect(res.body.data.length).to.equal(6)
})
it('Should fail without a correct uuid', async function () {

View File

@ -1,5 +1,6 @@
import './audio-only'
import './multiple-servers'
import './resumable-upload'
import './single-server'
import './video-captions'
import './video-change-ownership'

View File

@ -181,7 +181,7 @@ describe('Test multiple servers', function () {
thumbnailfile: 'thumbnail.jpg',
previewfile: 'preview.jpg'
}
await uploadVideo(servers[1].url, userAccessToken, videoAttributes)
await uploadVideo(servers[1].url, userAccessToken, videoAttributes, HttpStatusCode.OK_200, 'resumable')
// Transcoding
await waitJobs(servers)

View File

@ -0,0 +1,187 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import 'mocha'
import * as chai from 'chai'
import { pathExists, readdir, stat } from 'fs-extra'
import { join } from 'path'
import { HttpStatusCode } from '@shared/core-utils'
import {
buildAbsoluteFixturePath,
buildServerDirectory,
flushAndRunServer,
getMyUserInformation,
prepareResumableUpload,
sendDebugCommand,
sendResumableChunks,
ServerInfo,
setAccessTokensToServers,
setDefaultVideoChannel,
updateUser
} from '@shared/extra-utils'
import { MyUser, VideoPrivacy } from '@shared/models'
const expect = chai.expect
// Most classic resumable upload tests are done in other test suites
describe('Test resumable upload', function () {
const defaultFixture = 'video_short.mp4'
let server: ServerInfo
let rootId: number
async function buildSize (fixture: string, size?: number) {
if (size !== undefined) return size
const baseFixture = buildAbsoluteFixturePath(fixture)
return (await stat(baseFixture)).size
}
async function prepareUpload (sizeArg?: number) {
const size = await buildSize(defaultFixture, sizeArg)
const attributes = {
name: 'video',
channelId: server.videoChannel.id,
privacy: VideoPrivacy.PUBLIC,
fixture: defaultFixture
}
const mimetype = 'video/mp4'
const res = await prepareResumableUpload({ url: server.url, token: server.accessToken, attributes, size, mimetype })
return res.header['location'].split('?')[1]
}
async function sendChunks (options: {
pathUploadId: string
size?: number
expectedStatus?: HttpStatusCode
contentLength?: number
contentRange?: string
contentRangeBuilder?: (start: number, chunk: any) => string
}) {
const { pathUploadId, expectedStatus, contentLength, contentRangeBuilder } = options
const size = await buildSize(defaultFixture, options.size)
const absoluteFilePath = buildAbsoluteFixturePath(defaultFixture)
return sendResumableChunks({
url: server.url,
token: server.accessToken,
pathUploadId,
videoFilePath: absoluteFilePath,
size,
contentLength,
contentRangeBuilder,
specialStatus: expectedStatus
})
}
async function checkFileSize (uploadIdArg: string, expectedSize: number | null) {
const uploadId = uploadIdArg.replace(/^upload_id=/, '')
const subPath = join('tmp', 'resumable-uploads', uploadId)
const filePath = buildServerDirectory(server, subPath)
const exists = await pathExists(filePath)
if (expectedSize === null) {
expect(exists).to.be.false
return
}
expect(exists).to.be.true
expect((await stat(filePath)).size).to.equal(expectedSize)
}
async function countResumableUploads () {
const subPath = join('tmp', 'resumable-uploads')
const filePath = buildServerDirectory(server, subPath)
const files = await readdir(filePath)
return files.length
}
before(async function () {
this.timeout(30000)
server = await flushAndRunServer(1)
await setAccessTokensToServers([ server ])
await setDefaultVideoChannel([ server ])
const res = await getMyUserInformation(server.url, server.accessToken)
rootId = (res.body as MyUser).id
await updateUser({
url: server.url,
userId: rootId,
accessToken: server.accessToken,
videoQuota: 10_000_000
})
})
describe('Directory cleaning', function () {
it('Should correctly delete files after an upload', async function () {
const uploadId = await prepareUpload()
await sendChunks({ pathUploadId: uploadId })
expect(await countResumableUploads()).to.equal(0)
})
it('Should not delete files after an unfinished upload', async function () {
await prepareUpload()
expect(await countResumableUploads()).to.equal(2)
})
it('Should not delete recent uploads', async function () {
await sendDebugCommand(server.url, server.accessToken, { command: 'remove-dandling-resumable-uploads' })
expect(await countResumableUploads()).to.equal(2)
})
it('Should delete old uploads', async function () {
await sendDebugCommand(server.url, server.accessToken, { command: 'remove-dandling-resumable-uploads' })
expect(await countResumableUploads()).to.equal(0)
})
})
describe('Resumable upload and chunks', function () {
it('Should accept the same amount of chunks', async function () {
const uploadId = await prepareUpload()
await sendChunks({ pathUploadId: uploadId })
await checkFileSize(uploadId, null)
})
it('Should not accept more chunks than expected', async function () {
const size = 100
const uploadId = await prepareUpload(size)
await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.CONFLICT_409 })
await checkFileSize(uploadId, 0)
})
it('Should not accept more chunks than expected with an invalid content length/content range', async function () {
const uploadId = await prepareUpload(1500)
await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.BAD_REQUEST_400, contentLength: 1000 })
await checkFileSize(uploadId, 0)
})
it('Should not accept more chunks than expected with an invalid content length', async function () {
const uploadId = await prepareUpload(500)
const size = 1000
const contentRangeBuilder = start => `bytes ${start}-${start + size - 1}/${size}`
await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.BAD_REQUEST_400, contentRangeBuilder, contentLength: size })
await checkFileSize(uploadId, 0)
})
})
})

View File

@ -1,9 +1,9 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import 'mocha'
import * as chai from 'chai'
import { keyBy } from 'lodash'
import 'mocha'
import { VideoPrivacy } from '../../../../shared/models/videos'
import {
checkVideoFilesWereRemoved,
cleanupTests,
@ -28,430 +28,432 @@ import {
viewVideo,
wait
} from '../../../../shared/extra-utils'
import { VideoPrivacy } from '../../../../shared/models/videos'
import { HttpStatusCode } from '@shared/core-utils'
const expect = chai.expect
describe('Test a single server', function () {
let server: ServerInfo = null
let videoId = -1
let videoId2 = -1
let videoUUID = ''
let videosListBase: any[] = null
const getCheckAttributes = () => ({
name: 'my super name',
category: 2,
licence: 6,
language: 'zh',
nsfw: true,
description: 'my super description',
support: 'my super support text',
account: {
name: 'root',
host: 'localhost:' + server.port
},
isLocal: true,
duration: 5,
tags: [ 'tag1', 'tag2', 'tag3' ],
privacy: VideoPrivacy.PUBLIC,
commentsEnabled: true,
downloadEnabled: true,
channel: {
displayName: 'Main root channel',
name: 'root_channel',
description: '',
isLocal: true
},
fixture: 'video_short.webm',
files: [
{
resolution: 720,
size: 218910
}
]
})
function runSuite (mode: 'legacy' | 'resumable') {
let server: ServerInfo = null
let videoId = -1
let videoId2 = -1
let videoUUID = ''
let videosListBase: any[] = null
const updateCheckAttributes = () => ({
name: 'my super video updated',
category: 4,
licence: 2,
language: 'ar',
nsfw: false,
description: 'my super description updated',
support: 'my super support text updated',
account: {
name: 'root',
host: 'localhost:' + server.port
},
isLocal: true,
tags: [ 'tagup1', 'tagup2' ],
privacy: VideoPrivacy.PUBLIC,
duration: 5,
commentsEnabled: false,
downloadEnabled: false,
channel: {
name: 'root_channel',
displayName: 'Main root channel',
description: '',
isLocal: true
},
fixture: 'video_short3.webm',
files: [
{
resolution: 720,
size: 292677
}
]
})
before(async function () {
this.timeout(30000)
server = await flushAndRunServer(1)
await setAccessTokensToServers([ server ])
})
it('Should list video categories', async function () {
const res = await getVideoCategories(server.url)
const categories = res.body
expect(Object.keys(categories)).to.have.length.above(10)
expect(categories[11]).to.equal('News & Politics')
})
it('Should list video licences', async function () {
const res = await getVideoLicences(server.url)
const licences = res.body
expect(Object.keys(licences)).to.have.length.above(5)
expect(licences[3]).to.equal('Attribution - No Derivatives')
})
it('Should list video languages', async function () {
const res = await getVideoLanguages(server.url)
const languages = res.body
expect(Object.keys(languages)).to.have.length.above(5)
expect(languages['ru']).to.equal('Russian')
})
it('Should list video privacies', async function () {
const res = await getVideoPrivacies(server.url)
const privacies = res.body
expect(Object.keys(privacies)).to.have.length.at.least(3)
expect(privacies[3]).to.equal('Private')
})
it('Should not have videos', async function () {
const res = await getVideosList(server.url)
expect(res.body.total).to.equal(0)
expect(res.body.data).to.be.an('array')
expect(res.body.data.length).to.equal(0)
})
it('Should upload the video', async function () {
this.timeout(10000)
const videoAttributes = {
const getCheckAttributes = () => ({
name: 'my super name',
category: 2,
nsfw: true,
licence: 6,
tags: [ 'tag1', 'tag2', 'tag3' ]
}
const res = await uploadVideo(server.url, server.accessToken, videoAttributes)
expect(res.body.video).to.not.be.undefined
expect(res.body.video.id).to.equal(1)
expect(res.body.video.uuid).to.have.length.above(5)
language: 'zh',
nsfw: true,
description: 'my super description',
support: 'my super support text',
account: {
name: 'root',
host: 'localhost:' + server.port
},
isLocal: true,
duration: 5,
tags: [ 'tag1', 'tag2', 'tag3' ],
privacy: VideoPrivacy.PUBLIC,
commentsEnabled: true,
downloadEnabled: true,
channel: {
displayName: 'Main root channel',
name: 'root_channel',
description: '',
isLocal: true
},
fixture: 'video_short.webm',
files: [
{
resolution: 720,
size: 218910
}
]
})
videoId = res.body.video.id
videoUUID = res.body.video.uuid
})
it('Should get and seed the uploaded video', async function () {
this.timeout(5000)
const res = await getVideosList(server.url)
expect(res.body.total).to.equal(1)
expect(res.body.data).to.be.an('array')
expect(res.body.data.length).to.equal(1)
const video = res.body.data[0]
await completeVideoCheck(server.url, video, getCheckAttributes())
})
it('Should get the video by UUID', async function () {
this.timeout(5000)
const res = await getVideo(server.url, videoUUID)
const video = res.body
await completeVideoCheck(server.url, video, getCheckAttributes())
})
it('Should have the views updated', async function () {
this.timeout(20000)
await viewVideo(server.url, videoId)
await viewVideo(server.url, videoId)
await viewVideo(server.url, videoId)
await wait(1500)
await viewVideo(server.url, videoId)
await viewVideo(server.url, videoId)
await wait(1500)
await viewVideo(server.url, videoId)
await viewVideo(server.url, videoId)
// Wait the repeatable job
await wait(8000)
const res = await getVideo(server.url, videoId)
const video = res.body
expect(video.views).to.equal(3)
})
it('Should remove the video', async function () {
await removeVideo(server.url, server.accessToken, videoId)
await checkVideoFilesWereRemoved(videoUUID, 1)
})
it('Should not have videos', async function () {
const res = await getVideosList(server.url)
expect(res.body.total).to.equal(0)
expect(res.body.data).to.be.an('array')
expect(res.body.data).to.have.lengthOf(0)
})
it('Should upload 6 videos', async function () {
this.timeout(25000)
const videos = new Set([
'video_short.mp4', 'video_short.ogv', 'video_short.webm',
'video_short1.webm', 'video_short2.webm', 'video_short3.webm'
])
for (const video of videos) {
const videoAttributes = {
name: video + ' name',
description: video + ' description',
category: 2,
licence: 1,
language: 'en',
nsfw: true,
tags: [ 'tag1', 'tag2', 'tag3' ],
fixture: video
}
await uploadVideo(server.url, server.accessToken, videoAttributes)
}
})
it('Should have the correct durations', async function () {
const res = await getVideosList(server.url)
expect(res.body.total).to.equal(6)
const videos = res.body.data
expect(videos).to.be.an('array')
expect(videos).to.have.lengthOf(6)
const videosByName = keyBy<{ duration: number }>(videos, 'name')
expect(videosByName['video_short.mp4 name'].duration).to.equal(5)
expect(videosByName['video_short.ogv name'].duration).to.equal(5)
expect(videosByName['video_short.webm name'].duration).to.equal(5)
expect(videosByName['video_short1.webm name'].duration).to.equal(10)
expect(videosByName['video_short2.webm name'].duration).to.equal(5)
expect(videosByName['video_short3.webm name'].duration).to.equal(5)
})
it('Should have the correct thumbnails', async function () {
const res = await getVideosList(server.url)
const videos = res.body.data
// For the next test
videosListBase = videos
for (const video of videos) {
const videoName = video.name.replace(' name', '')
await testImage(server.url, videoName, video.thumbnailPath)
}
})
it('Should list only the two first videos', async function () {
const res = await getVideosListPagination(server.url, 0, 2, 'name')
const videos = res.body.data
expect(res.body.total).to.equal(6)
expect(videos.length).to.equal(2)
expect(videos[0].name).to.equal(videosListBase[0].name)
expect(videos[1].name).to.equal(videosListBase[1].name)
})
it('Should list only the next three videos', async function () {
const res = await getVideosListPagination(server.url, 2, 3, 'name')
const videos = res.body.data
expect(res.body.total).to.equal(6)
expect(videos.length).to.equal(3)
expect(videos[0].name).to.equal(videosListBase[2].name)
expect(videos[1].name).to.equal(videosListBase[3].name)
expect(videos[2].name).to.equal(videosListBase[4].name)
})
it('Should list the last video', async function () {
const res = await getVideosListPagination(server.url, 5, 6, 'name')
const videos = res.body.data
expect(res.body.total).to.equal(6)
expect(videos.length).to.equal(1)
expect(videos[0].name).to.equal(videosListBase[5].name)
})
it('Should not have the total field', async function () {
const res = await getVideosListPagination(server.url, 5, 6, 'name', true)
const videos = res.body.data
expect(res.body.total).to.not.exist
expect(videos.length).to.equal(1)
expect(videos[0].name).to.equal(videosListBase[5].name)
})
it('Should list and sort by name in descending order', async function () {
const res = await getVideosListSort(server.url, '-name')
const videos = res.body.data
expect(res.body.total).to.equal(6)
expect(videos.length).to.equal(6)
expect(videos[0].name).to.equal('video_short.webm name')
expect(videos[1].name).to.equal('video_short.ogv name')
expect(videos[2].name).to.equal('video_short.mp4 name')
expect(videos[3].name).to.equal('video_short3.webm name')
expect(videos[4].name).to.equal('video_short2.webm name')
expect(videos[5].name).to.equal('video_short1.webm name')
videoId = videos[3].uuid
videoId2 = videos[5].uuid
})
it('Should list and sort by trending in descending order', async function () {
const res = await getVideosListPagination(server.url, 0, 2, '-trending')
const videos = res.body.data
expect(res.body.total).to.equal(6)
expect(videos.length).to.equal(2)
})
it('Should list and sort by hotness in descending order', async function () {
const res = await getVideosListPagination(server.url, 0, 2, '-hot')
const videos = res.body.data
expect(res.body.total).to.equal(6)
expect(videos.length).to.equal(2)
})
it('Should list and sort by best in descending order', async function () {
const res = await getVideosListPagination(server.url, 0, 2, '-best')
const videos = res.body.data
expect(res.body.total).to.equal(6)
expect(videos.length).to.equal(2)
})
it('Should update a video', async function () {
const attributes = {
const updateCheckAttributes = () => ({
name: 'my super video updated',
category: 4,
licence: 2,
language: 'ar',
nsfw: false,
description: 'my super description updated',
support: 'my super support text updated',
account: {
name: 'root',
host: 'localhost:' + server.port
},
isLocal: true,
tags: [ 'tagup1', 'tagup2' ],
privacy: VideoPrivacy.PUBLIC,
duration: 5,
commentsEnabled: false,
downloadEnabled: false,
tags: [ 'tagup1', 'tagup2' ]
}
await updateVideo(server.url, server.accessToken, videoId, attributes)
})
channel: {
name: 'root_channel',
displayName: 'Main root channel',
description: '',
isLocal: true
},
fixture: 'video_short3.webm',
files: [
{
resolution: 720,
size: 292677
}
]
})
it('Should filter by tags and category', async function () {
const res1 = await getVideosWithFilters(server.url, { tagsAllOf: [ 'tagup1', 'tagup2' ], categoryOneOf: [ 4 ] })
expect(res1.body.total).to.equal(1)
expect(res1.body.data[0].name).to.equal('my super video updated')
before(async function () {
this.timeout(30000)
const res2 = await getVideosWithFilters(server.url, { tagsAllOf: [ 'tagup1', 'tagup2' ], categoryOneOf: [ 3 ] })
expect(res2.body.total).to.equal(0)
})
server = await flushAndRunServer(1)
it('Should have the video updated', async function () {
this.timeout(60000)
await setAccessTokensToServers([ server ])
})
const res = await getVideo(server.url, videoId)
const video = res.body
it('Should list video categories', async function () {
const res = await getVideoCategories(server.url)
await completeVideoCheck(server.url, video, updateCheckAttributes())
})
const categories = res.body
expect(Object.keys(categories)).to.have.length.above(10)
it('Should update only the tags of a video', async function () {
const attributes = {
tags: [ 'supertag', 'tag1', 'tag2' ]
}
await updateVideo(server.url, server.accessToken, videoId, attributes)
expect(categories[11]).to.equal('News & Politics')
})
const res = await getVideo(server.url, videoId)
const video = res.body
it('Should list video licences', async function () {
const res = await getVideoLicences(server.url)
await completeVideoCheck(server.url, video, Object.assign(updateCheckAttributes(), attributes))
})
const licences = res.body
expect(Object.keys(licences)).to.have.length.above(5)
it('Should update only the description of a video', async function () {
const attributes = {
description: 'hello everybody'
}
await updateVideo(server.url, server.accessToken, videoId, attributes)
expect(licences[3]).to.equal('Attribution - No Derivatives')
})
const res = await getVideo(server.url, videoId)
const video = res.body
it('Should list video languages', async function () {
const res = await getVideoLanguages(server.url)
const expectedAttributes = Object.assign(updateCheckAttributes(), { tags: [ 'supertag', 'tag1', 'tag2' ] }, attributes)
await completeVideoCheck(server.url, video, expectedAttributes)
})
const languages = res.body
expect(Object.keys(languages)).to.have.length.above(5)
it('Should like a video', async function () {
await rateVideo(server.url, server.accessToken, videoId, 'like')
expect(languages['ru']).to.equal('Russian')
})
const res = await getVideo(server.url, videoId)
const video = res.body
it('Should list video privacies', async function () {
const res = await getVideoPrivacies(server.url)
expect(video.likes).to.equal(1)
expect(video.dislikes).to.equal(0)
})
const privacies = res.body
expect(Object.keys(privacies)).to.have.length.at.least(3)
it('Should dislike the same video', async function () {
await rateVideo(server.url, server.accessToken, videoId, 'dislike')
expect(privacies[3]).to.equal('Private')
})
const res = await getVideo(server.url, videoId)
const video = res.body
it('Should not have videos', async function () {
const res = await getVideosList(server.url)
expect(video.likes).to.equal(0)
expect(video.dislikes).to.equal(1)
})
expect(res.body.total).to.equal(0)
expect(res.body.data).to.be.an('array')
expect(res.body.data.length).to.equal(0)
})
it('Should sort by originallyPublishedAt', async function () {
{
it('Should upload the video', async function () {
this.timeout(10000)
const videoAttributes = {
name: 'my super name',
category: 2,
nsfw: true,
licence: 6,
tags: [ 'tag1', 'tag2', 'tag3' ]
}
const res = await uploadVideo(server.url, server.accessToken, videoAttributes, HttpStatusCode.OK_200, mode)
expect(res.body.video).to.not.be.undefined
expect(res.body.video.id).to.equal(1)
expect(res.body.video.uuid).to.have.length.above(5)
videoId = res.body.video.id
videoUUID = res.body.video.uuid
})
it('Should get and seed the uploaded video', async function () {
this.timeout(5000)
const res = await getVideosList(server.url)
expect(res.body.total).to.equal(1)
expect(res.body.data).to.be.an('array')
expect(res.body.data.length).to.equal(1)
const video = res.body.data[0]
await completeVideoCheck(server.url, video, getCheckAttributes())
})
it('Should get the video by UUID', async function () {
this.timeout(5000)
const res = await getVideo(server.url, videoUUID)
const video = res.body
await completeVideoCheck(server.url, video, getCheckAttributes())
})
it('Should have the views updated', async function () {
this.timeout(20000)
await viewVideo(server.url, videoId)
await viewVideo(server.url, videoId)
await viewVideo(server.url, videoId)
await wait(1500)
await viewVideo(server.url, videoId)
await viewVideo(server.url, videoId)
await wait(1500)
await viewVideo(server.url, videoId)
await viewVideo(server.url, videoId)
// Wait the repeatable job
await wait(8000)
const res = await getVideo(server.url, videoId)
const video = res.body
expect(video.views).to.equal(3)
})
it('Should remove the video', async function () {
await removeVideo(server.url, server.accessToken, videoId)
await checkVideoFilesWereRemoved(videoUUID, 1)
})
it('Should not have videos', async function () {
const res = await getVideosList(server.url)
expect(res.body.total).to.equal(0)
expect(res.body.data).to.be.an('array')
expect(res.body.data).to.have.lengthOf(0)
})
it('Should upload 6 videos', async function () {
this.timeout(25000)
const videos = new Set([
'video_short.mp4', 'video_short.ogv', 'video_short.webm',
'video_short1.webm', 'video_short2.webm', 'video_short3.webm'
])
for (const video of videos) {
const videoAttributes = {
name: video + ' name',
description: video + ' description',
category: 2,
licence: 1,
language: 'en',
nsfw: true,
tags: [ 'tag1', 'tag2', 'tag3' ],
fixture: video
}
await uploadVideo(server.url, server.accessToken, videoAttributes, HttpStatusCode.OK_200, mode)
}
})
it('Should have the correct durations', async function () {
const res = await getVideosList(server.url)
expect(res.body.total).to.equal(6)
const videos = res.body.data
expect(videos).to.be.an('array')
expect(videos).to.have.lengthOf(6)
const videosByName = keyBy<{ duration: number }>(videos, 'name')
expect(videosByName['video_short.mp4 name'].duration).to.equal(5)
expect(videosByName['video_short.ogv name'].duration).to.equal(5)
expect(videosByName['video_short.webm name'].duration).to.equal(5)
expect(videosByName['video_short1.webm name'].duration).to.equal(10)
expect(videosByName['video_short2.webm name'].duration).to.equal(5)
expect(videosByName['video_short3.webm name'].duration).to.equal(5)
})
it('Should have the correct thumbnails', async function () {
const res = await getVideosList(server.url)
const videos = res.body.data
// For the next test
videosListBase = videos
for (const video of videos) {
const videoName = video.name.replace(' name', '')
await testImage(server.url, videoName, video.thumbnailPath)
}
})
it('Should list only the two first videos', async function () {
const res = await getVideosListPagination(server.url, 0, 2, 'name')
const videos = res.body.data
expect(res.body.total).to.equal(6)
expect(videos.length).to.equal(2)
expect(videos[0].name).to.equal(videosListBase[0].name)
expect(videos[1].name).to.equal(videosListBase[1].name)
})
it('Should list only the next three videos', async function () {
const res = await getVideosListPagination(server.url, 2, 3, 'name')
const videos = res.body.data
expect(res.body.total).to.equal(6)
expect(videos.length).to.equal(3)
expect(videos[0].name).to.equal(videosListBase[2].name)
expect(videos[1].name).to.equal(videosListBase[3].name)
expect(videos[2].name).to.equal(videosListBase[4].name)
})
it('Should list the last video', async function () {
const res = await getVideosListPagination(server.url, 5, 6, 'name')
const videos = res.body.data
expect(res.body.total).to.equal(6)
expect(videos.length).to.equal(1)
expect(videos[0].name).to.equal(videosListBase[5].name)
})
it('Should not have the total field', async function () {
const res = await getVideosListPagination(server.url, 5, 6, 'name', true)
const videos = res.body.data
expect(res.body.total).to.not.exist
expect(videos.length).to.equal(1)
expect(videos[0].name).to.equal(videosListBase[5].name)
})
it('Should list and sort by name in descending order', async function () {
const res = await getVideosListSort(server.url, '-name')
const videos = res.body.data
expect(res.body.total).to.equal(6)
expect(videos.length).to.equal(6)
expect(videos[0].name).to.equal('video_short.webm name')
expect(videos[1].name).to.equal('video_short.ogv name')
expect(videos[2].name).to.equal('video_short.mp4 name')
expect(videos[3].name).to.equal('video_short3.webm name')
expect(videos[4].name).to.equal('video_short2.webm name')
expect(videos[5].name).to.equal('video_short1.webm name')
videoId = videos[3].uuid
videoId2 = videos[5].uuid
})
it('Should list and sort by trending in descending order', async function () {
const res = await getVideosListPagination(server.url, 0, 2, '-trending')
const videos = res.body.data
expect(res.body.total).to.equal(6)
expect(videos.length).to.equal(2)
})
it('Should list and sort by hotness in descending order', async function () {
const res = await getVideosListPagination(server.url, 0, 2, '-hot')
const videos = res.body.data
expect(res.body.total).to.equal(6)
expect(videos.length).to.equal(2)
})
it('Should list and sort by best in descending order', async function () {
const res = await getVideosListPagination(server.url, 0, 2, '-best')
const videos = res.body.data
expect(res.body.total).to.equal(6)
expect(videos.length).to.equal(2)
})
it('Should update a video', async function () {
const attributes = {
name: 'my super video updated',
category: 4,
licence: 2,
language: 'ar',
nsfw: false,
description: 'my super description updated',
commentsEnabled: false,
downloadEnabled: false,
tags: [ 'tagup1', 'tagup2' ]
}
await updateVideo(server.url, server.accessToken, videoId, attributes)
})
it('Should filter by tags and category', async function () {
const res1 = await getVideosWithFilters(server.url, { tagsAllOf: [ 'tagup1', 'tagup2' ], categoryOneOf: [ 4 ] })
expect(res1.body.total).to.equal(1)
expect(res1.body.data[0].name).to.equal('my super video updated')
const res2 = await getVideosWithFilters(server.url, { tagsAllOf: [ 'tagup1', 'tagup2' ], categoryOneOf: [ 3 ] })
expect(res2.body.total).to.equal(0)
})
it('Should have the video updated', async function () {
this.timeout(60000)
const res = await getVideo(server.url, videoId)
const video = res.body
await completeVideoCheck(server.url, video, updateCheckAttributes())
})
it('Should update only the tags of a video', async function () {
const attributes = {
tags: [ 'supertag', 'tag1', 'tag2' ]
}
await updateVideo(server.url, server.accessToken, videoId, attributes)
const res = await getVideo(server.url, videoId)
const video = res.body
await completeVideoCheck(server.url, video, Object.assign(updateCheckAttributes(), attributes))
})
it('Should update only the description of a video', async function () {
const attributes = {
description: 'hello everybody'
}
await updateVideo(server.url, server.accessToken, videoId, attributes)
const res = await getVideo(server.url, videoId)
const video = res.body
const expectedAttributes = Object.assign(updateCheckAttributes(), { tags: [ 'supertag', 'tag1', 'tag2' ] }, attributes)
await completeVideoCheck(server.url, video, expectedAttributes)
})
it('Should like a video', async function () {
await rateVideo(server.url, server.accessToken, videoId, 'like')
const res = await getVideo(server.url, videoId)
const video = res.body
expect(video.likes).to.equal(1)
expect(video.dislikes).to.equal(0)
})
it('Should dislike the same video', async function () {
await rateVideo(server.url, server.accessToken, videoId, 'dislike')
const res = await getVideo(server.url, videoId)
const video = res.body
expect(video.likes).to.equal(0)
expect(video.dislikes).to.equal(1)
})
it('Should sort by originallyPublishedAt', async function () {
{
const now = new Date()
const attributes = { originallyPublishedAt: now.toISOString() }
@ -483,10 +485,18 @@ describe('Test a single server', function () {
expect(names[4]).to.equal('video_short.ogv name')
expect(names[5]).to.equal('video_short.mp4 name')
}
}
})
after(async function () {
await cleanupTests([ server ])
})
}
describe('Legacy upload', function () {
runSuite('legacy')
})
after(async function () {
await cleanupTests([ server ])
describe('Resumable upload', function () {
runSuite('resumable')
})
})

View File

@ -361,106 +361,117 @@ describe('Test video transcoding', function () {
describe('Audio upload', function () {
before(async function () {
await updateCustomSubConfig(servers[1].url, servers[1].accessToken, {
transcoding: {
hls: { enabled: true },
webtorrent: { enabled: true },
resolutions: {
'0p': false,
'240p': false,
'360p': false,
'480p': false,
'720p': false,
'1080p': false,
'1440p': false,
'2160p': false
function runSuite (mode: 'legacy' | 'resumable') {
before(async function () {
await updateCustomSubConfig(servers[1].url, servers[1].accessToken, {
transcoding: {
hls: { enabled: true },
webtorrent: { enabled: true },
resolutions: {
'0p': false,
'240p': false,
'360p': false,
'480p': false,
'720p': false,
'1080p': false,
'1440p': false,
'2160p': false
}
}
}
})
})
})
it('Should merge an audio file with the preview file', async function () {
this.timeout(60_000)
it('Should merge an audio file with the preview file', async function () {
this.timeout(60_000)
const videoAttributesArg = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' }
await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg)
const videoAttributesArg = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' }
await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg, HttpStatusCode.OK_200, mode)
await waitJobs(servers)
await waitJobs(servers)
for (const server of servers) {
const res = await getVideosList(server.url)
for (const server of servers) {
const res = await getVideosList(server.url)
const video = res.body.data.find(v => v.name === 'audio_with_preview')
const res2 = await getVideo(server.url, video.id)
const videoDetails: VideoDetails = res2.body
const video = res.body.data.find(v => v.name === 'audio_with_preview')
const res2 = await getVideo(server.url, video.id)
const videoDetails: VideoDetails = res2.body
expect(videoDetails.files).to.have.lengthOf(1)
expect(videoDetails.files).to.have.lengthOf(1)
await makeGetRequest({ url: server.url, path: videoDetails.thumbnailPath, statusCodeExpected: HttpStatusCode.OK_200 })
await makeGetRequest({ url: server.url, path: videoDetails.previewPath, statusCodeExpected: HttpStatusCode.OK_200 })
await makeGetRequest({ url: server.url, path: videoDetails.thumbnailPath, statusCodeExpected: HttpStatusCode.OK_200 })
await makeGetRequest({ url: server.url, path: videoDetails.previewPath, statusCodeExpected: HttpStatusCode.OK_200 })
const magnetUri = videoDetails.files[0].magnetUri
expect(magnetUri).to.contain('.mp4')
}
})
it('Should upload an audio file and choose a default background image', async function () {
this.timeout(60_000)
const videoAttributesArg = { name: 'audio_without_preview', fixture: 'sample.ogg' }
await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg)
await waitJobs(servers)
for (const server of servers) {
const res = await getVideosList(server.url)
const video = res.body.data.find(v => v.name === 'audio_without_preview')
const res2 = await getVideo(server.url, video.id)
const videoDetails = res2.body
expect(videoDetails.files).to.have.lengthOf(1)
await makeGetRequest({ url: server.url, path: videoDetails.thumbnailPath, statusCodeExpected: HttpStatusCode.OK_200 })
await makeGetRequest({ url: server.url, path: videoDetails.previewPath, statusCodeExpected: HttpStatusCode.OK_200 })
const magnetUri = videoDetails.files[0].magnetUri
expect(magnetUri).to.contain('.mp4')
}
})
it('Should upload an audio file and create an audio version only', async function () {
this.timeout(60_000)
await updateCustomSubConfig(servers[1].url, servers[1].accessToken, {
transcoding: {
hls: { enabled: true },
webtorrent: { enabled: true },
resolutions: {
'0p': true,
'240p': false,
'360p': false
}
const magnetUri = videoDetails.files[0].magnetUri
expect(magnetUri).to.contain('.mp4')
}
})
const videoAttributesArg = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' }
const resVideo = await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg)
it('Should upload an audio file and choose a default background image', async function () {
this.timeout(60_000)
await waitJobs(servers)
const videoAttributesArg = { name: 'audio_without_preview', fixture: 'sample.ogg' }
await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg, HttpStatusCode.OK_200, mode)
for (const server of servers) {
const res2 = await getVideo(server.url, resVideo.body.video.id)
const videoDetails: VideoDetails = res2.body
await waitJobs(servers)
for (const files of [ videoDetails.files, videoDetails.streamingPlaylists[0].files ]) {
expect(files).to.have.lengthOf(2)
expect(files.find(f => f.resolution.id === 0)).to.not.be.undefined
for (const server of servers) {
const res = await getVideosList(server.url)
const video = res.body.data.find(v => v.name === 'audio_without_preview')
const res2 = await getVideo(server.url, video.id)
const videoDetails = res2.body
expect(videoDetails.files).to.have.lengthOf(1)
await makeGetRequest({ url: server.url, path: videoDetails.thumbnailPath, statusCodeExpected: HttpStatusCode.OK_200 })
await makeGetRequest({ url: server.url, path: videoDetails.previewPath, statusCodeExpected: HttpStatusCode.OK_200 })
const magnetUri = videoDetails.files[0].magnetUri
expect(magnetUri).to.contain('.mp4')
}
}
})
await updateConfigForTranscoding(servers[1])
it('Should upload an audio file and create an audio version only', async function () {
this.timeout(60_000)
await updateCustomSubConfig(servers[1].url, servers[1].accessToken, {
transcoding: {
hls: { enabled: true },
webtorrent: { enabled: true },
resolutions: {
'0p': true,
'240p': false,
'360p': false
}
}
})
const videoAttributesArg = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' }
const resVideo = await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg, HttpStatusCode.OK_200, mode)
await waitJobs(servers)
for (const server of servers) {
const res2 = await getVideo(server.url, resVideo.body.video.id)
const videoDetails: VideoDetails = res2.body
for (const files of [ videoDetails.files, videoDetails.streamingPlaylists[0].files ]) {
expect(files).to.have.lengthOf(2)
expect(files.find(f => f.resolution.id === 0)).to.not.be.undefined
}
}
await updateConfigForTranscoding(servers[1])
})
}
describe('Legacy upload', function () {
runSuite('legacy')
})
describe('Resumable upload', function () {
runSuite('resumable')
})
})

View File

@ -19,6 +19,9 @@ import { MPlugin, MServer, MServerBlocklist } from '@server/types/models/server'
import { MVideoImportDefault } from '@server/types/models/video/video-import'
import { MVideoPlaylistElement, MVideoPlaylistElementVideoUrlPlaylistPrivacy } from '@server/types/models/video/video-playlist-element'
import { MAccountVideoRateAccountVideo } from '@server/types/models/video/video-rate'
import { HttpMethod } from '@shared/core-utils/miscs/http-methods'
import { VideoCreate } from '@shared/models'
import { File as UploadXFile, Metadata } from '@uploadx/core'
import { RegisteredPlugin } from '../../lib/plugins/plugin-manager'
import {
MAccountDefault,
@ -37,86 +40,125 @@ import {
MVideoThumbnail,
MVideoWithRights
} from '../../types/models'
declare module 'express' {
export interface Request {
query: any
method: HttpMethod
}
// Upload using multer or uploadx middleware
export type MulterOrUploadXFile = UploadXFile | Express.Multer.File
export type UploadFiles = {
[fieldname: string]: MulterOrUploadXFile[]
} | MulterOrUploadXFile[]
// Partial object used by some functions to check the file mimetype/extension
export type UploadFileForCheck = {
originalname: string
mimetype: string
}
export type UploadFilesForCheck = {
[fieldname: string]: UploadFileForCheck[]
} | UploadFileForCheck[]
// Upload file with a duration added by our middleware
export type VideoUploadFile = Pick<Express.Multer.File, 'path' | 'filename' | 'size'> & {
duration: number
}
// Extends Metadata property of UploadX object
export type UploadXFileMetadata = Metadata & VideoCreate & {
previewfile: Express.Multer.File[]
thumbnailfile: Express.Multer.File[]
}
// Our custom UploadXFile object using our custom metadata
export type CustomUploadXFile <T extends Metadata> = UploadXFile & { metadata: T }
export type EnhancedUploadXFile = CustomUploadXFile<UploadXFileMetadata> & {
duration: number
path: string
filename: string
}
// Extends locals property from Response
interface Response {
locals: PeerTubeLocals
locals: {
videoAll?: MVideoFullLight
onlyImmutableVideo?: MVideoImmutable
onlyVideo?: MVideoThumbnail
onlyVideoWithRights?: MVideoWithRights
videoId?: MVideoIdThumbnail
videoLive?: MVideoLive
videoShare?: MVideoShareActor
videoFile?: MVideoFile
videoFileResumable?: EnhancedUploadXFile
videoImport?: MVideoImportDefault
videoBlacklist?: MVideoBlacklist
videoCaption?: MVideoCaptionVideo
abuse?: MAbuseReporter
abuseMessage?: MAbuseMessage
videoStreamingPlaylist?: MStreamingPlaylist
videoChannel?: MChannelBannerAccountDefault
videoPlaylistFull?: MVideoPlaylistFull
videoPlaylistSummary?: MVideoPlaylistFullSummary
videoPlaylistElement?: MVideoPlaylistElement
videoPlaylistElementAP?: MVideoPlaylistElementVideoUrlPlaylistPrivacy
accountVideoRate?: MAccountVideoRateAccountVideo
videoCommentFull?: MCommentOwnerVideoReply
videoCommentThread?: MComment
follow?: MActorFollowActorsDefault
subscription?: MActorFollowActorsDefaultSubscription
nextOwner?: MAccountDefault
videoChangeOwnership?: MVideoChangeOwnershipFull
account?: MAccountDefault
actorUrl?: MActorUrl
actorFull?: MActorFull
user?: MUserDefault
server?: MServer
videoRedundancy?: MVideoRedundancyVideo
accountBlock?: MAccountBlocklist
serverBlock?: MServerBlocklist
oauth?: {
token: MOAuthTokenUser
}
signature?: {
actor: MActorAccountChannelId
}
authenticated?: boolean
registeredPlugin?: RegisteredPlugin
externalAuth?: RegisterServerAuthExternalOptions
plugin?: MPlugin
}
}
}
interface PeerTubeLocals {
videoAll?: MVideoFullLight
onlyImmutableVideo?: MVideoImmutable
onlyVideo?: MVideoThumbnail
onlyVideoWithRights?: MVideoWithRights
videoId?: MVideoIdThumbnail
videoLive?: MVideoLive
videoShare?: MVideoShareActor
videoFile?: MVideoFile
videoImport?: MVideoImportDefault
videoBlacklist?: MVideoBlacklist
videoCaption?: MVideoCaptionVideo
abuse?: MAbuseReporter
abuseMessage?: MAbuseMessage
videoStreamingPlaylist?: MStreamingPlaylist
videoChannel?: MChannelBannerAccountDefault
videoPlaylistFull?: MVideoPlaylistFull
videoPlaylistSummary?: MVideoPlaylistFullSummary
videoPlaylistElement?: MVideoPlaylistElement
videoPlaylistElementAP?: MVideoPlaylistElementVideoUrlPlaylistPrivacy
accountVideoRate?: MAccountVideoRateAccountVideo
videoCommentFull?: MCommentOwnerVideoReply
videoCommentThread?: MComment
follow?: MActorFollowActorsDefault
subscription?: MActorFollowActorsDefaultSubscription
nextOwner?: MAccountDefault
videoChangeOwnership?: MVideoChangeOwnershipFull
account?: MAccountDefault
actorUrl?: MActorUrl
actorFull?: MActorFull
user?: MUserDefault
server?: MServer
videoRedundancy?: MVideoRedundancyVideo
accountBlock?: MAccountBlocklist
serverBlock?: MServerBlocklist
oauth?: {
token: MOAuthTokenUser
}
signature?: {
actor: MActorAccountChannelId
}
authenticated?: boolean
registeredPlugin?: RegisteredPlugin
externalAuth?: RegisterServerAuthExternalOptions
plugin?: MPlugin
}

View File

@ -0,0 +1,21 @@
/** HTTP request method to indicate the desired action to be performed for a given resource. */
export enum HttpMethod {
/** The CONNECT method establishes a tunnel to the server identified by the target resource. */
CONNECT = 'CONNECT',
/** The DELETE method deletes the specified resource. */
DELETE = 'DELETE',
/** The GET method requests a representation of the specified resource. Requests using GET should only retrieve data. */
GET = 'GET',
/** The HEAD method asks for a response identical to that of a GET request, but without the response body. */
HEAD = 'HEAD',
/** The OPTIONS method is used to describe the communication options for the target resource. */
OPTIONS = 'OPTIONS',
/** The PATCH method is used to apply partial modifications to a resource. */
PATCH = 'PATCH',
/** The POST method is used to submit an entity to the specified resource */
POST = 'POST',
/** The PUT method replaces all current representations of the target resource with the request payload. */
PUT = 'PUT',
/** The TRACE method performs a message loop-back test along the path to the target resource. */
TRACE = 'TRACE'
}

View File

@ -2,3 +2,4 @@ export * from './date'
export * from './miscs'
export * from './types'
export * from './http-error-codes'
export * from './http-methods'

View File

@ -1,5 +1,6 @@
import { makeGetRequest } from '../requests/requests'
import { makeGetRequest, makePostBodyRequest } from '../requests/requests'
import { HttpStatusCode } from '../../core-utils/miscs/http-error-codes'
import { SendDebugCommand } from '@shared/models'
function getDebug (url: string, token: string) {
const path = '/api/v1/server/debug'
@ -12,8 +13,21 @@ function getDebug (url: string, token: string) {
})
}
function sendDebugCommand (url: string, token: string, body: SendDebugCommand) {
const path = '/api/v1/server/debug/run-command'
return makePostBodyRequest({
url,
path,
token,
fields: body,
statusCodeExpected: HttpStatusCode.NO_CONTENT_204
})
}
// ---------------------------------------------------------------------------
export {
getDebug
getDebug,
sendDebugCommand
}

View File

@ -274,7 +274,7 @@ async function reRunServer (server: ServerInfo, configOverride?: any) {
}
async function checkTmpIsEmpty (server: ServerInfo) {
await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls' ])
await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls', 'resumable-uploads' ])
if (await pathExists(join('test' + server.internalServerNumber, 'tmp', 'hls'))) {
await checkDirectoryIsEmpty(server, 'tmp/hls')

View File

@ -5,7 +5,7 @@ import { VideoChannelUpdate } from '../../models/videos/channel/video-channel-up
import { VideoChannelCreate } from '../../models/videos/channel/video-channel-create.model'
import { makeDeleteRequest, makeGetRequest, updateImageRequest } from '../requests/requests'
import { ServerInfo } from '../server/servers'
import { User } from '../../models/users/user.model'
import { MyUser, User } from '../../models/users/user.model'
import { getMyUserInformation } from '../users/users'
import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
@ -170,6 +170,12 @@ function setDefaultVideoChannel (servers: ServerInfo[]) {
return Promise.all(tasks)
}
async function getDefaultVideoChannel (url: string, token: string) {
const res = await getMyUserInformation(url, token)
return (res.body as MyUser).videoChannels[0].id
}
// ---------------------------------------------------------------------------
export {
@ -181,5 +187,6 @@ export {
deleteVideoChannel,
getVideoChannel,
setDefaultVideoChannel,
deleteVideoChannelImage
deleteVideoChannelImage,
getDefaultVideoChannel
}

View File

@ -1,7 +1,8 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */
import { expect } from 'chai'
import { pathExists, readdir, readFile } from 'fs-extra'
import { createReadStream, pathExists, readdir, readFile, stat } from 'fs-extra'
import got, { Response as GotResponse } from 'got/dist/source'
import * as parseTorrent from 'parse-torrent'
import { extname, join } from 'path'
import * as request from 'supertest'
@ -42,6 +43,7 @@ type VideoAttributes = {
channelId?: number
privacy?: VideoPrivacy
fixture?: string
support?: string
thumbnailfile?: string
previewfile?: string
scheduleUpdate?: {
@ -364,8 +366,13 @@ async function checkVideoFilesWereRemoved (
}
}
async function uploadVideo (url: string, accessToken: string, videoAttributesArg: VideoAttributes, specialStatus = HttpStatusCode.OK_200) {
const path = '/api/v1/videos/upload'
async function uploadVideo (
url: string,
accessToken: string,
videoAttributesArg: VideoAttributes,
specialStatus = HttpStatusCode.OK_200,
mode: 'legacy' | 'resumable' = 'legacy'
) {
let defaultChannelId = '1'
try {
@ -391,61 +398,9 @@ async function uploadVideo (url: string, accessToken: string, videoAttributesArg
fixture: 'video_short.webm'
}, videoAttributesArg)
const req = request(url)
.post(path)
.set('Accept', 'application/json')
.set('Authorization', 'Bearer ' + accessToken)
.field('name', attributes.name)
.field('nsfw', JSON.stringify(attributes.nsfw))
.field('commentsEnabled', JSON.stringify(attributes.commentsEnabled))
.field('downloadEnabled', JSON.stringify(attributes.downloadEnabled))
.field('waitTranscoding', JSON.stringify(attributes.waitTranscoding))
.field('privacy', attributes.privacy.toString())
.field('channelId', attributes.channelId)
if (attributes.support !== undefined) {
req.field('support', attributes.support)
}
if (attributes.description !== undefined) {
req.field('description', attributes.description)
}
if (attributes.language !== undefined) {
req.field('language', attributes.language.toString())
}
if (attributes.category !== undefined) {
req.field('category', attributes.category.toString())
}
if (attributes.licence !== undefined) {
req.field('licence', attributes.licence.toString())
}
const tags = attributes.tags || []
for (let i = 0; i < tags.length; i++) {
req.field('tags[' + i + ']', attributes.tags[i])
}
if (attributes.thumbnailfile !== undefined) {
req.attach('thumbnailfile', buildAbsoluteFixturePath(attributes.thumbnailfile))
}
if (attributes.previewfile !== undefined) {
req.attach('previewfile', buildAbsoluteFixturePath(attributes.previewfile))
}
if (attributes.scheduleUpdate) {
req.field('scheduleUpdate[updateAt]', attributes.scheduleUpdate.updateAt)
if (attributes.scheduleUpdate.privacy) {
req.field('scheduleUpdate[privacy]', attributes.scheduleUpdate.privacy)
}
}
if (attributes.originallyPublishedAt !== undefined) {
req.field('originallyPublishedAt', attributes.originallyPublishedAt)
}
const res = await req.attach('videofile', buildAbsoluteFixturePath(attributes.fixture))
.expect(specialStatus)
const res = mode === 'legacy'
? await buildLegacyUpload(url, accessToken, attributes, specialStatus)
: await buildResumeUpload(url, accessToken, attributes, specialStatus)
// Wait torrent generation
if (specialStatus === HttpStatusCode.OK_200) {
@ -461,6 +416,154 @@ async function uploadVideo (url: string, accessToken: string, videoAttributesArg
return res
}
function checkUploadVideoParam (
url: string,
token: string,
attributes: Partial<VideoAttributes>,
specialStatus = HttpStatusCode.OK_200,
mode: 'legacy' | 'resumable' = 'legacy'
) {
return mode === 'legacy'
? buildLegacyUpload(url, token, attributes, specialStatus)
: buildResumeUpload(url, token, attributes, specialStatus)
}
async function buildLegacyUpload (url: string, token: string, attributes: VideoAttributes, specialStatus = HttpStatusCode.OK_200) {
const path = '/api/v1/videos/upload'
const req = request(url)
.post(path)
.set('Accept', 'application/json')
.set('Authorization', 'Bearer ' + token)
buildUploadReq(req, attributes)
if (attributes.fixture !== undefined) {
req.attach('videofile', buildAbsoluteFixturePath(attributes.fixture))
}
return req.expect(specialStatus)
}
async function buildResumeUpload (url: string, token: string, attributes: VideoAttributes, specialStatus = HttpStatusCode.OK_200) {
let size = 0
let videoFilePath: string
let mimetype = 'video/mp4'
if (attributes.fixture) {
videoFilePath = buildAbsoluteFixturePath(attributes.fixture)
size = (await stat(videoFilePath)).size
if (videoFilePath.endsWith('.mkv')) {
mimetype = 'video/x-matroska'
} else if (videoFilePath.endsWith('.webm')) {
mimetype = 'video/webm'
}
}
const initializeSessionRes = await prepareResumableUpload({ url, token, attributes, size, mimetype })
const initStatus = initializeSessionRes.status
if (videoFilePath && initStatus === HttpStatusCode.CREATED_201) {
const locationHeader = initializeSessionRes.header['location']
expect(locationHeader).to.not.be.undefined
const pathUploadId = locationHeader.split('?')[1]
return sendResumableChunks({ url, token, pathUploadId, videoFilePath, size, specialStatus })
}
const expectedInitStatus = specialStatus === HttpStatusCode.OK_200
? HttpStatusCode.CREATED_201
: specialStatus
expect(initStatus).to.equal(expectedInitStatus)
return initializeSessionRes
}
async function prepareResumableUpload (options: {
url: string
token: string
attributes: VideoAttributes
size: number
mimetype: string
}) {
const { url, token, attributes, size, mimetype } = options
const path = '/api/v1/videos/upload-resumable'
const req = request(url)
.post(path)
.set('Authorization', 'Bearer ' + token)
.set('X-Upload-Content-Type', mimetype)
.set('X-Upload-Content-Length', size.toString())
buildUploadReq(req, attributes)
if (attributes.fixture) {
req.field('filename', attributes.fixture)
}
return req
}
function sendResumableChunks (options: {
url: string
token: string
pathUploadId: string
videoFilePath: string
size: number
specialStatus?: HttpStatusCode
contentLength?: number
contentRangeBuilder?: (start: number, chunk: any) => string
}) {
const { url, token, pathUploadId, videoFilePath, size, specialStatus, contentLength, contentRangeBuilder } = options
const expectedStatus = specialStatus || HttpStatusCode.OK_200
const path = '/api/v1/videos/upload-resumable'
let start = 0
const readable = createReadStream(videoFilePath, { highWaterMark: 8 * 1024 })
return new Promise<GotResponse>((resolve, reject) => {
readable.on('data', async function onData (chunk) {
readable.pause()
const headers = {
'Authorization': 'Bearer ' + token,
'Content-Type': 'application/octet-stream',
'Content-Range': contentRangeBuilder
? contentRangeBuilder(start, chunk)
: `bytes ${start}-${start + chunk.length - 1}/${size}`,
'Content-Length': contentLength ? contentLength + '' : chunk.length + ''
}
const res = await got({
url,
method: 'put',
headers,
path: path + '?' + pathUploadId,
body: chunk,
responseType: 'json',
throwHttpErrors: false
})
start += chunk.length
if (res.statusCode === expectedStatus) {
return resolve(res)
}
if (res.statusCode !== HttpStatusCode.PERMANENT_REDIRECT_308) {
readable.off('data', onData)
return reject(new Error('Incorrect transient behaviour sending intermediary chunks'))
}
readable.resume()
})
})
}
function updateVideo (
url: string,
accessToken: string,
@ -749,11 +852,13 @@ export {
getVideoWithToken,
getVideosList,
removeAllVideos,
checkUploadVideoParam,
getVideosListPagination,
getVideosListSort,
removeVideo,
getVideosListWithToken,
uploadVideo,
sendResumableChunks,
getVideosWithFilters,
uploadRandomVideoOnServers,
updateVideo,
@ -767,5 +872,50 @@ export {
getMyVideosWithFilter,
uploadVideoAndGetId,
getLocalIdByUUID,
getVideoIdFromUUID
getVideoIdFromUUID,
prepareResumableUpload
}
// ---------------------------------------------------------------------------
function buildUploadReq (req: request.Test, attributes: VideoAttributes) {
for (const key of [ 'name', 'support', 'channelId', 'description', 'originallyPublishedAt' ]) {
if (attributes[key] !== undefined) {
req.field(key, attributes[key])
}
}
for (const key of [ 'nsfw', 'commentsEnabled', 'downloadEnabled', 'waitTranscoding' ]) {
if (attributes[key] !== undefined) {
req.field(key, JSON.stringify(attributes[key]))
}
}
for (const key of [ 'language', 'privacy', 'category', 'licence' ]) {
if (attributes[key] !== undefined) {
req.field(key, attributes[key].toString())
}
}
const tags = attributes.tags || []
for (let i = 0; i < tags.length; i++) {
req.field('tags[' + i + ']', attributes.tags[i])
}
for (const key of [ 'thumbnailfile', 'previewfile' ]) {
if (attributes[key] !== undefined) {
req.attach(key, buildAbsoluteFixturePath(attributes[key]))
}
}
if (attributes.scheduleUpdate) {
if (attributes.scheduleUpdate.updateAt) {
req.field('scheduleUpdate[updateAt]', attributes.scheduleUpdate.updateAt)
}
if (attributes.scheduleUpdate.privacy) {
req.field('scheduleUpdate[privacy]', attributes.scheduleUpdate.privacy)
}
}
}

View File

@ -1,3 +1,7 @@
export interface Debug {
ip: string
}
export interface SendDebugCommand {
command: 'remove-dandling-resumable-uploads'
}

View File

@ -125,7 +125,7 @@ tags:
Redundancy is part of the inter-server solidarity that PeerTube fosters.
Manage the list of instances you wish to help by seeding their videos according
to the policy of video selection of your choice. Note that you have a similar functionality
to mirror individual videos, see `Video Mirroring`.
to mirror individual videos, see [video mirroring](#tag/Video-Mirroring).
externalDocs:
url: https://docs.joinpeertube.org/admin-following-instances?id=instances-redundancy
- name: Plugins
@ -139,6 +139,50 @@ tags:
- name: Video
description: |
Operations dealing with listing, uploading, fetching or modifying videos.
- name: Video Upload
description: |
Operations dealing with adding video or audio. PeerTube supports two upload modes, and three import modes.
### Upload
- [_legacy_](#tag/Video-Upload/paths/~1videos~1upload/post), where the video file is sent in a single request
- [_resumable_](#tag/Video-Upload/paths/~1videos~1upload-resumable/post), where the video file is sent in chunks
You can upload videos more reliably by using the resumable variant. Its protocol lets
you resume an upload operation after a network interruption or other transmission failure,
saving time and bandwidth in the event of network failures.
Favor using resumable uploads in any of the following cases:
- You are transferring large files
- The likelihood of a network interruption is high
- Uploads are originating from a device with a low-bandwidth or unstable Internet connection,
such as a mobile device
### Import
- _URL_-based: where the URL points to any service supported by [youtube-dl](https://ytdl-org.github.io/youtube-dl/)
- _magnet_-based: where the URI resolves to a BitTorrent ressource containing a single supported video file
- _torrent_-based: where the metainfo file resolves to a BitTorrent ressource containing a single supported video file
The import function is practical when the desired video/audio is available online. It makes PeerTube
download it for you, saving you as much bandwidth and avoiding any instability or limitation your network might have.
- name: Video Captions
description: Operations dealing with listing, adding and removing closed captions of a video.
- name: Video Channels
description: Operations dealing with the creation, modification and listing of videos within a channel.
- name: Video Comments
description: >
Operations dealing with comments to a video. Comments are organized in threads: adding a
comment in response to the video starts a thread, adding a reply to a comment adds it to
its root comment thread.
- name: Video Blocks
description: Operations dealing with blocking videos (removing them from view and preventing interactions).
- name: Video Rates
description: Like/dislike a video.
- name: Video Playlists
description: Operations dealing with playlists of videos. Playlists are bound to users and/or channels.
- name: Feeds
description: Server syndication feeds
- name: Search
description: |
The search helps to find _videos_ or _channels_ from within the instance and beyond.
@ -148,27 +192,11 @@ tags:
Administrators can also enable the use of a remote search system, indexing
videos and channels not could be not federated by the instance.
- name: Video Comments
description: >
Operations dealing with comments to a video. Comments are organized in
threads.
- name: Video Playlists
description: >
Operations dealing with playlists of videos. Playlists are bound to users
and/or channels.
- name: Video Channels
description: >
Operations dealing with the creation, modification and listing of videos within a channel.
- name: Video Blocks
description: >
Operations dealing with blocking videos (removing them from view and
preventing interactions).
- name: Video Rates
description: >
Like/dislike a video.
- name: Feeds
description: >
Server syndication feeds
- name: Video Mirroring
description: |
PeerTube instances can mirror videos from one another, and help distribute some videos.
For importing videos as your own, refer to [video imports](#tag/Video-Upload/paths/~1videos~1imports/post).
x-tagGroups:
- name: Accounts
tags:
@ -181,6 +209,7 @@ x-tagGroups:
- name: Videos
tags:
- Video
- Video Upload
- Video Captions
- Video Channels
- Video Comments
@ -1347,10 +1376,12 @@ paths:
/videos/upload:
post:
summary: Upload a video
description: Uses a single request to upload a video.
security:
- OAuth2: []
tags:
- Video
- Video Upload
responses:
'200':
description: successful operation
@ -1380,80 +1411,7 @@ paths:
content:
multipart/form-data:
schema:
type: object
properties:
videofile:
description: Video file
type: string
format: binary
channelId:
description: Channel id that will contain this video
type: integer
thumbnailfile:
description: Video thumbnail file
type: string
format: binary
previewfile:
description: Video preview file
type: string
format: binary
privacy:
$ref: '#/components/schemas/VideoPrivacySet'
category:
description: Video category
type: integer
example: 4
licence:
description: Video licence
type: integer
example: 2
language:
description: Video language
type: string
description:
description: Video description
type: string
waitTranscoding:
description: Whether or not we wait transcoding before publish the video
type: boolean
support:
description: A text tell the audience how to support the video creator
example: Please support my work on <insert crowdfunding plateform>! <3
type: string
nsfw:
description: Whether or not this video contains sensitive content
type: boolean
name:
description: Video name
type: string
minLength: 3
maxLength: 120
tags:
description: Video tags (maximum 5 tags each between 2 and 30 characters)
type: array
minItems: 1
maxItems: 5
uniqueItems: true
items:
type: string
minLength: 2
maxLength: 30
commentsEnabled:
description: Enable or disable comments for this video
type: boolean
downloadEnabled:
description: Enable or disable downloading for this video
type: boolean
originallyPublishedAt:
description: Date when the content was originally published
type: string
format: date-time
scheduleUpdate:
$ref: '#/components/schemas/VideoScheduledUpdate'
required:
- videofile
- channelId
- name
$ref: '#/components/schemas/VideoUploadRequestLegacy'
encoding:
videofile:
contentType: video/mp4, video/webm, video/ogg, video/avi, video/quicktime, video/x-msvideo, video/x-flv, video/x-matroska, application/octet-stream
@ -1490,6 +1448,164 @@ paths:
--form videofile=@"$FILE_PATH" \
--form channelId=$CHANNEL_ID \
--form name="$NAME"
/videos/upload-resumable:
post:
summary: Initialize the resumable upload of a video
description: Uses [a resumable protocol](https://github.com/kukhariev/node-uploadx/blob/master/proto.md) to initialize the upload of a video
security:
- OAuth2: []
tags:
- Video
- Video Upload
parameters:
- name: X-Upload-Content-Length
in: header
schema:
type: number
example: 2469036
required: true
description: Number of bytes that will be uploaded in subsequent requests. Set this value to the size of the file you are uploading.
- name: X-Upload-Content-Type
in: header
schema:
type: string
format: mimetype
example: video/mp4
required: true
description: MIME type of the file that you are uploading. Depending on your instance settings, acceptable values might vary.
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/VideoUploadRequestResumable'
responses:
'200':
description: file already exists, send a [`resume`](https://github.com/kukhariev/node-uploadx/blob/master/proto.md) request instead
'201':
description: created
headers:
Location:
schema:
type: string
format: url
example: /api/v1/videos/upload-resumable?upload_id=471e97554f21dec3b8bb5d4602939c51
Content-Length:
schema:
type: number
example: 0
'400':
description: invalid file field, schedule date or parameter
'413':
description: video file too large, due to quota, absolute max file size or concurrent partial upload limit
'415':
description: video type unsupported
put:
summary: Send chunk for the resumable upload of a video
description: Uses [a resumable protocol](https://github.com/kukhariev/node-uploadx/blob/master/proto.md) to continue, pause or resume the upload of a video
security:
- OAuth2: []
tags:
- Video
- Video Upload
parameters:
- name: upload_id
in: path
required: true
description: |
Created session id to proceed with. If you didn't send chunks in the last 12 hours, it is
not valid anymore and you need to initialize a new upload.
schema:
type: string
- name: Content-Range
in: header
schema:
type: string
example: bytes 0-262143/2469036
required: true
description: |
Specifies the bytes in the file that the request is uploading.
For example, a value of `bytes 0-262143/1000000` shows that the request is sending the first
262144 bytes (256 x 1024) in a 2,469,036 byte file.
- name: Content-Length
in: header
schema:
type: number
example: 262144
required: true
description: |
Size of the chunk that the request is sending.
The chunk size __must be a multiple of 256 KB__, and unlike [Google Resumable](https://developers.google.com/youtube/v3/guides/using_resumable_upload_protocol)
doesn't mandate for chunks to have the same size throughout the upload sequence.
Remember that larger chunks are more efficient. PeerTube's web client uses chunks varying from
1048576 bytes (~1MB) and increases or reduces size depending on connection health.
requestBody:
content:
application/octet-stream:
schema:
type: string
format: binary
responses:
'200':
description: last chunk received
headers:
Content-Length:
schema:
type: number
content:
application/json:
schema:
$ref: '#/components/schemas/VideoUploadResponse'
'308':
description: resume incomplete
headers:
Range:
schema:
type: string
example: bytes=0-262143
Content-Length:
schema:
type: number
example: 0
'403':
description: video didn't pass upload filter
'413':
description: video file too large, due to quota or max body size limit set by the reverse-proxy
'422':
description: video unreadable
delete:
summary: Cancel the resumable upload of a video, deleting any data uploaded so far
description: Uses [a resumable protocol](https://github.com/kukhariev/node-uploadx/blob/master/proto.md) to cancel the upload of a video
security:
- OAuth2: []
tags:
- Video
- Video Upload
parameters:
- name: upload_id
in: path
required: true
description: |
Created session id to proceed with. If you didn't send chunks in the last 12 hours, it is
not valid anymore and the upload session has already been deleted with its data ;-)
schema:
type: string
- name: Content-Length
in: header
required: true
schema:
type: number
example: 0
responses:
'204':
description: upload cancelled
headers:
Content-Length:
schema:
type: number
example: 0
/videos/imports:
post:
summary: Import a video
@ -1498,6 +1614,7 @@ paths:
- OAuth2: []
tags:
- Video
- Video Upload
requestBody:
content:
multipart/form-data:
@ -1688,7 +1805,7 @@ paths:
/videos/live/{id}:
get:
summary: Get a live information
summary: Get information about a live
security:
- OAuth2: []
tags:
@ -1704,7 +1821,7 @@ paths:
schema:
$ref: '#/components/schemas/LiveVideoResponse'
put:
summary: Update a live information
summary: Update information about a live
security:
- OAuth2: []
tags:
@ -3940,6 +4057,7 @@ components:
oneOf:
- type: string
- type: array
maxItems: 5
items:
type: string
style: form
@ -4636,7 +4754,7 @@ components:
message:
type: string
minLength: 2
maxLength: 3000
maxLength: 3000
byModerator:
type: boolean
createdAt:
@ -5229,6 +5347,7 @@ components:
PredefinedAbuseReasons:
description: Reason categories that help triage reports
type: array
maxItems: 8
items:
type: string
enum:
@ -5298,6 +5417,103 @@ components:
id:
type: integer
example: 37
VideoUploadRequestCommon:
properties:
name:
description: Video name
type: string
channelId:
description: Channel id that will contain this video
type: integer
privacy:
$ref: '#/components/schemas/VideoPrivacySet'
category:
description: Video category
type: integer
example: 4
licence:
description: Video licence
type: integer
example: 2
language:
description: Video language
type: string
description:
description: Video description
type: string
waitTranscoding:
description: Whether or not we wait transcoding before publish the video
type: boolean
support:
description: A text tell the audience how to support the video creator
example: Please support my work on <insert crowdfunding plateform>! <3
type: string
nsfw:
description: Whether or not this video contains sensitive content
type: boolean
tags:
description: Video tags (maximum 5 tags each between 2 and 30 characters)
type: array
minItems: 1
maxItems: 5
uniqueItems: true
items:
type: string
minLength: 2
maxLength: 30
commentsEnabled:
description: Enable or disable comments for this video
type: boolean
downloadEnabled:
description: Enable or disable downloading for this video
type: boolean
originallyPublishedAt:
description: Date when the content was originally published
type: string
format: date-time
scheduleUpdate:
$ref: '#/components/schemas/VideoScheduledUpdate'
thumbnailfile:
description: Video thumbnail file
type: string
format: binary
previewfile:
description: Video preview file
type: string
format: binary
required:
- channelId
- name
VideoUploadRequestLegacy:
allOf:
- $ref: '#/components/schemas/VideoUploadRequestCommon'
- type: object
required:
- videofile
properties:
videofile:
description: Video file
type: string
format: binary
VideoUploadRequestResumable:
allOf:
- $ref: '#/components/schemas/VideoUploadRequestCommon'
- type: object
required:
- filename
properties:
filename:
description: Video filename including extension
type: string
format: filename
thumbnailfile:
description: Video thumbnail file
type: string
format: binary
previewfile:
description: Video preview file
type: string
format: binary
VideoUploadResponse:
properties:
video:

View File

@ -78,6 +78,13 @@ server {
try_files /dev/null @api;
}
location = /api/v1/videos/upload-resumable {
client_max_body_size 0;
proxy_request_buffering off;
try_files /dev/null @api;
}
location = /api/v1/videos/upload {
limit_except POST HEAD { deny all; }

View File

@ -1061,6 +1061,15 @@
resolved "https://registry.yarnpkg.com/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz#aa58042711d6e3275dd37dc597e5d31e8c290a44"
integrity sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q==
"@uploadx/core@^4.4.0":
version "4.4.0"
resolved "https://registry.yarnpkg.com/@uploadx/core/-/core-4.4.0.tgz#27ea2b0d28125e81a6bdd65637dc5c7829306cc7"
integrity sha512-dU0oDURYR5RvuAzf63EL9e/fCY4OOQKOs237UTbZDulbRbiyxwEZR+IpRYYr3hKRjjij03EF/Y5j54VGkebAKg==
dependencies:
bytes "^3.1.0"
debug "^4.3.1"
multiparty "^4.2.2"
abbrev@1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8"
@ -1794,7 +1803,7 @@ busboy@^0.2.11:
dicer "0.2.5"
readable-stream "1.1.x"
bytes@3.1.0, bytes@^3.0.0:
bytes@3.1.0, bytes@^3.0.0, bytes@^3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6"
integrity sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==
@ -4098,6 +4107,17 @@ http-errors@~1.7.2:
statuses ">= 1.5.0 < 2"
toidentifier "1.0.0"
http-errors@~1.8.0:
version "1.8.0"
resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.8.0.tgz#75d1bbe497e1044f51e4ee9e704a62f28d336507"
integrity sha512-4I8r0C5JDhT5VkvI47QktDW75rNlGVsUf/8hzjCC/wkWI/jdTRmBb9aI7erSG82r1bjKY3F6k28WnsVxB1C73A==
dependencies:
depd "~1.1.2"
inherits "2.0.4"
setprototypeof "1.2.0"
statuses ">= 1.5.0 < 2"
toidentifier "1.0.0"
"http-node@github:feross/http-node#webtorrent":
version "1.2.0"
resolved "https://codeload.github.com/feross/http-node/tar.gz/342ef8624495343ffd050bd0808b3750cf0e3974"
@ -5567,6 +5587,15 @@ multimatch@^5.0.0:
arrify "^2.0.1"
minimatch "^3.0.4"
multiparty@^4.2.2:
version "4.2.2"
resolved "https://registry.yarnpkg.com/multiparty/-/multiparty-4.2.2.tgz#bee5fb5737247628d39dab4979ffd6d57bf60ef6"
integrity sha512-NtZLjlvsjcoGrzojtwQwn/Tm90aWJ6XXtPppYF4WmOk/6ncdwMMKggFY2NlRRN9yiCEIVxpOfPWahVEG2HAG8Q==
dependencies:
http-errors "~1.8.0"
safe-buffer "5.2.1"
uid-safe "2.1.5"
multistream@^4.0.1, multistream@^4.1.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/multistream/-/multistream-4.1.0.tgz#7bf00dfd119556fbc153cff3de4c6d477909f5a8"
@ -6656,6 +6685,11 @@ random-access-storage@^1.1.1:
dependencies:
inherits "^2.0.3"
random-bytes@~1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/random-bytes/-/random-bytes-1.0.0.tgz#4f68a1dc0ae58bd3fb95848c30324db75d64360b"
integrity sha1-T2ih3Arli9P7lYSMMDJNt11kNgs=
random-iterate@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/random-iterate/-/random-iterate-1.0.1.tgz#f7d97d92dee6665ec5f6da08c7f963cad4b2ac99"
@ -7040,7 +7074,7 @@ safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1:
resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==
safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@~5.2.0:
safe-buffer@5.2.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@~5.2.0:
version "5.2.1"
resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
@ -7186,6 +7220,11 @@ setprototypeof@1.1.1:
resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.1.tgz#7e95acb24aa92f5885e0abef5ba131330d4ae683"
integrity sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==
setprototypeof@1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.2.0.tgz#66c9a24a73f9fc28cbe66b09fed3d33dcaf1b424"
integrity sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==
shebang-command@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-1.2.0.tgz#44aac65b695b03398968c39f363fee5deafdf1ea"
@ -8139,6 +8178,13 @@ uc.micro@^1.0.1, uc.micro@^1.0.5:
resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-1.0.6.tgz#9c411a802a409a91fc6cf74081baba34b24499ac"
integrity sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==
uid-safe@2.1.5:
version "2.1.5"
resolved "https://registry.yarnpkg.com/uid-safe/-/uid-safe-2.1.5.tgz#2b3d5c7240e8fc2e58f8aa269e5ee49c0857bd3a"
integrity sha512-KPHm4VL5dDXKz01UuEd88Df+KzynaohSL9fBh096KWAxSKZQDI2uBrVqtvRM4rwrIrRRKsdLNML/lnaaVSRioA==
dependencies:
random-bytes "~1.0.0"
uint64be@^2.0.2:
version "2.0.2"
resolved "https://registry.yarnpkg.com/uint64be/-/uint64be-2.0.2.tgz#ef4a179752fe8f9ddaa29544ecfc13490031e8e5"