Import magnets with webtorrent

This commit is contained in:
Chocobozzz 2018-08-06 17:13:39 +02:00
parent 788487140c
commit ce33919c24
23 changed files with 648 additions and 142 deletions

View File

@ -26,34 +26,23 @@ export class VideoImportService {
private serverService: ServerService
) {}
importVideo (targetUrl: string, video: VideoUpdate): Observable<VideoImport> {
importVideoUrl (targetUrl: string, video: VideoUpdate): Observable<VideoImport> {
const url = VideoImportService.BASE_VIDEO_IMPORT_URL
const language = video.language || null
const licence = video.licence || null
const category = video.category || null
const description = video.description || null
const support = video.support || null
const scheduleUpdate = video.scheduleUpdate || null
const body: VideoImportCreate = {
targetUrl,
const body = this.buildImportVideoObject(video)
body.targetUrl = targetUrl
name: video.name,
category,
licence,
language,
support,
description,
channelId: video.channelId,
privacy: video.privacy,
tags: video.tags,
nsfw: video.nsfw,
waitTranscoding: video.waitTranscoding,
commentsEnabled: video.commentsEnabled,
thumbnailfile: video.thumbnailfile,
previewfile: video.previewfile,
scheduleUpdate
}
const data = objectToFormData(body)
return this.authHttp.post<VideoImport>(url, data)
.pipe(catchError(res => this.restExtractor.handleError(res)))
}
importVideoTorrent (target: string | Blob, video: VideoUpdate): Observable<VideoImport> {
const url = VideoImportService.BASE_VIDEO_IMPORT_URL
const body: VideoImportCreate = this.buildImportVideoObject(video)
if (typeof target === 'string') body.magnetUri = target
else body.torrentfile = target
const data = objectToFormData(body)
return this.authHttp.post<VideoImport>(url, data)
@ -73,6 +62,33 @@ export class VideoImportService {
)
}
private buildImportVideoObject (video: VideoUpdate): VideoImportCreate {
const language = video.language || null
const licence = video.licence || null
const category = video.category || null
const description = video.description || null
const support = video.support || null
const scheduleUpdate = video.scheduleUpdate || null
return {
name: video.name,
category,
licence,
language,
support,
description,
channelId: video.channelId,
privacy: video.privacy,
tags: video.tags,
nsfw: video.nsfw,
waitTranscoding: video.waitTranscoding,
commentsEnabled: video.commentsEnabled,
thumbnailfile: video.thumbnailfile,
previewfile: video.previewfile,
scheduleUpdate
}
}
private extractVideoImports (result: ResultList<VideoImport>): Observable<ResultList<VideoImport>> {
return this.serverService.localeObservable
.pipe(

View File

@ -0,0 +1,60 @@
<div *ngIf="!hasImportedVideo" class="upload-video-container">
<div class="import-video-torrent">
<div class="icon icon-upload"></div>
<div class="form-group">
<label i18n for="magnetUri">Magnet URI</label>
<my-help
helpType="custom" i18n-customHtml
customHtml="You can import any torrent file that points to a mp4 file. You should make sure you have diffusion rights over the content it points to, otherwise it could cause legal trouble to yourself and your instance."
></my-help>
<input type="text" id="magnetUri" [(ngModel)]="magnetUri" />
</div>
<div class="form-group">
<label i18n for="first-step-channel">Channel</label>
<div class="peertube-select-container">
<select id="first-step-channel" [(ngModel)]="firstStepChannelId">
<option *ngFor="let channel of userVideoChannels" [value]="channel.id">{{ channel.label }}</option>
</select>
</div>
</div>
<div class="form-group">
<label i18n for="first-step-privacy">Privacy</label>
<div class="peertube-select-container">
<select id="first-step-privacy" [(ngModel)]="firstStepPrivacyId">
<option *ngFor="let privacy of videoPrivacies" [value]="privacy.id">{{ privacy.label }}</option>
</select>
</div>
</div>
<input
type="button" i18n-value value="Import"
[disabled]="!isMagnetUrlValid() || isImportingVideo" (click)="importVideo()"
/>
</div>
</div>
<div *ngIf="hasImportedVideo" class="alert alert-info" i18n>
Congratulations, the video will be imported with BitTorrent! You can already add information about this video.
</div>
<!-- Hidden because we want to load the component -->
<form [hidden]="!hasImportedVideo" novalidate [formGroup]="form">
<my-video-edit
[form]="form" [formErrors]="formErrors" [videoCaptions]="videoCaptions" [schedulePublicationPossible]="false"
[validationMessages]="validationMessages" [videoPrivacies]="videoPrivacies" [userVideoChannels]="userVideoChannels"
></my-video-edit>
<div class="submit-container">
<div class="submit-button"
(click)="updateSecondStep()"
[ngClass]="{ disabled: !form.valid || isUpdatingVideo === true }"
>
<span class="icon icon-validate"></span>
<input type="button" i18n-value value="Update" />
</div>
</div>
</form>

View File

@ -0,0 +1,37 @@
@import 'variables';
@import 'mixins';
$width-size: 190px;
.peertube-select-container {
@include peertube-select-container($width-size);
}
.import-video-torrent {
display: flex;
flex-direction: column;
align-items: center;
.icon.icon-upload {
@include icon(90px);
margin-bottom: 25px;
cursor: default;
background-image: url('../../../../assets/images/video/upload.svg');
}
input[type=text] {
@include peertube-input-text($width-size);
display: block;
}
input[type=button] {
@include peertube-button;
@include orange-button;
width: $width-size;
margin-top: 30px;
}
}

View File

@ -0,0 +1,132 @@
import { Component, EventEmitter, OnInit, Output } from '@angular/core'
import { Router } from '@angular/router'
import { NotificationsService } from 'angular2-notifications'
import { VideoPrivacy, VideoUpdate } from '../../../../../../shared/models/videos'
import { AuthService, ServerService } from '../../../core'
import { VideoService } from '../../../shared/video/video.service'
import { I18n } from '@ngx-translate/i18n-polyfill'
import { LoadingBarService } from '@ngx-loading-bar/core'
import { VideoSend } from '@app/videos/+video-edit/video-add-components/video-send'
import { CanComponentDeactivate } from '@app/shared/guards/can-deactivate-guard.service'
import { VideoEdit } from '@app/shared/video/video-edit.model'
import { FormValidatorService } from '@app/shared'
import { VideoCaptionService } from '@app/shared/video-caption'
import { VideoImportService } from '@app/shared/video-import'
@Component({
selector: 'my-video-import-torrent',
templateUrl: './video-import-torrent.component.html',
styleUrls: [
'../shared/video-edit.component.scss',
'./video-import-torrent.component.scss'
]
})
export class VideoImportTorrentComponent extends VideoSend implements OnInit, CanComponentDeactivate {
@Output() firstStepDone = new EventEmitter<string>()
videoFileName: string
magnetUri = ''
isImportingVideo = false
hasImportedVideo = false
isUpdatingVideo = false
video: VideoEdit
protected readonly DEFAULT_VIDEO_PRIVACY = VideoPrivacy.PRIVATE
constructor (
protected formValidatorService: FormValidatorService,
protected loadingBar: LoadingBarService,
protected notificationsService: NotificationsService,
protected authService: AuthService,
protected serverService: ServerService,
protected videoService: VideoService,
protected videoCaptionService: VideoCaptionService,
private router: Router,
private videoImportService: VideoImportService,
private i18n: I18n
) {
super()
}
ngOnInit () {
super.ngOnInit()
}
canDeactivate () {
return { canDeactivate: true }
}
isMagnetUrlValid () {
return !!this.magnetUri
}
importVideo () {
this.isImportingVideo = true
const videoUpdate: VideoUpdate = {
privacy: this.firstStepPrivacyId,
waitTranscoding: false,
commentsEnabled: true,
channelId: this.firstStepChannelId
}
this.loadingBar.start()
this.videoImportService.importVideoTorrent(this.magnetUri, videoUpdate).subscribe(
res => {
this.loadingBar.complete()
this.firstStepDone.emit(res.video.name)
this.isImportingVideo = false
this.hasImportedVideo = true
this.video = new VideoEdit(Object.assign(res.video, {
commentsEnabled: videoUpdate.commentsEnabled,
support: null,
thumbnailUrl: null,
previewUrl: null
}))
this.hydrateFormFromVideo()
},
err => {
this.loadingBar.complete()
this.isImportingVideo = false
this.notificationsService.error(this.i18n('Error'), err.message)
}
)
}
updateSecondStep () {
if (this.checkForm() === false) {
return
}
this.video.patch(this.form.value)
this.isUpdatingVideo = true
// Update the video
this.updateVideoAndCaptions(this.video)
.subscribe(
() => {
this.isUpdatingVideo = false
this.notificationsService.success(this.i18n('Success'), this.i18n('Video to import updated.'))
this.router.navigate([ '/my-account', 'video-imports' ])
},
err => {
this.isUpdatingVideo = false
this.notificationsService.error(this.i18n('Error'), err.message)
console.error(err)
}
)
}
private hydrateFormFromVideo () {
this.form.patchValue(this.video.toFormPatch())
}
}

View File

@ -1,5 +1,5 @@
<div *ngIf="!hasImportedVideo" class="upload-video-container">
<div class="import-video">
<div class="import-video-url">
<div class="icon icon-upload"></div>
<div class="form-group">

View File

@ -7,7 +7,7 @@ $width-size: 190px;
@include peertube-select-container($width-size);
}
.import-video {
.import-video-url {
display: flex;
flex-direction: column;
align-items: center;

View File

@ -74,7 +74,7 @@ export class VideoImportUrlComponent extends VideoSend implements OnInit, CanCom
this.loadingBar.start()
this.videoImportService.importVideo(this.targetUrl, videoUpdate).subscribe(
this.videoImportService.importVideoUrl(this.targetUrl, videoUpdate).subscribe(
res => {
this.loadingBar.complete()
this.firstStepDone.emit(res.video.name)

View File

@ -10,8 +10,12 @@
<my-video-upload #videoUpload (firstStepDone)="onFirstStepDone('upload', $event)"></my-video-upload>
</tab>
<tab *ngIf="isVideoImportEnabled()" i18n-heading heading="Import with URL">
<tab *ngIf="isVideoImportHttpEnabled()" i18n-heading heading="Import with URL">
<my-video-import-url #videoImportUrl (firstStepDone)="onFirstStepDone('import-url', $event)"></my-video-import-url>
</tab>
<tab *ngIf="isVideoImportTorrentEnabled()" i18n-heading heading="Import with torrent">
<my-video-import-torrent #videoImportTorrent (firstStepDone)="onFirstStepDone('import-torrent', $event)"></my-video-import-torrent>
</tab>
</tabset>
</div>

View File

@ -3,6 +3,7 @@ import { CanComponentDeactivate } from '@app/shared/guards/can-deactivate-guard.
import { VideoImportUrlComponent } from '@app/videos/+video-edit/video-add-components/video-import-url.component'
import { VideoUploadComponent } from '@app/videos/+video-edit/video-add-components/video-upload.component'
import { ServerService } from '@app/core'
import { VideoImportTorrentComponent } from '@app/videos/+video-edit/video-add-components/video-import-torrent.component'
@Component({
selector: 'my-videos-add',
@ -12,15 +13,16 @@ import { ServerService } from '@app/core'
export class VideoAddComponent implements CanComponentDeactivate {
@ViewChild('videoUpload') videoUpload: VideoUploadComponent
@ViewChild('videoImportUrl') videoImportUrl: VideoImportUrlComponent
@ViewChild('videoImportTorrent') videoImportTorrent: VideoImportTorrentComponent
secondStepType: 'upload' | 'import-url'
secondStepType: 'upload' | 'import-url' | 'import-torrent'
videoName: string
constructor (
private serverService: ServerService
) {}
onFirstStepDone (type: 'upload' | 'import-url', videoName: string) {
onFirstStepDone (type: 'upload' | 'import-url' | 'import-torrent', videoName: string) {
this.secondStepType = type
this.videoName = videoName
}
@ -28,11 +30,16 @@ export class VideoAddComponent implements CanComponentDeactivate {
canDeactivate () {
if (this.secondStepType === 'upload') return this.videoUpload.canDeactivate()
if (this.secondStepType === 'import-url') return this.videoImportUrl.canDeactivate()
if (this.secondStepType === 'import-torrent') return this.videoImportTorrent.canDeactivate()
return { canDeactivate: true }
}
isVideoImportEnabled () {
isVideoImportHttpEnabled () {
return this.serverService.getConfig().import.videos.http.enabled
}
isVideoImportTorrentEnabled () {
return this.serverService.getConfig().import.videos.http.enabled
}
}

View File

@ -7,6 +7,7 @@ import { VideoAddComponent } from './video-add.component'
import { CanDeactivateGuard } from '../../shared/guards/can-deactivate-guard.service'
import { VideoUploadComponent } from '@app/videos/+video-edit/video-add-components/video-upload.component'
import { VideoImportUrlComponent } from '@app/videos/+video-edit/video-add-components/video-import-url.component'
import { VideoImportTorrentComponent } from '@app/videos/+video-edit/video-add-components/video-import-torrent.component'
@NgModule({
imports: [
@ -18,7 +19,8 @@ import { VideoImportUrlComponent } from '@app/videos/+video-edit/video-add-compo
declarations: [
VideoAddComponent,
VideoUploadComponent,
VideoImportUrlComponent
VideoImportUrlComponent,
VideoImportTorrentComponent
],
exports: [
VideoAddComponent

View File

@ -134,6 +134,7 @@
"uuid": "^3.1.0",
"validator": "^10.2.0",
"webfinger.js": "^2.6.6",
"webtorrent": "^0.100.0",
"winston": "3.0.0",
"ws": "^5.0.0",
"youtube-dl": "^1.12.2"
@ -187,7 +188,6 @@
"tslint": "^5.7.0",
"tslint-config-standard": "^7.0.0",
"typescript": "^2.5.2",
"webtorrent": "^0.100.0",
"xliff": "^3.0.1"
},
"scripty": {

View File

@ -1,3 +1,4 @@
import * as magnetUtil from 'magnet-uri'
import * as express from 'express'
import { auditLoggerFactory, VideoImportAuditView } from '../../../helpers/audit-logger'
import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares'
@ -13,6 +14,10 @@ import { VideoImportModel } from '../../../models/video/video-import'
import { JobQueue } from '../../../lib/job-queue/job-queue'
import { processImage } from '../../../helpers/image-utils'
import { join } from 'path'
import { isArray } from '../../../helpers/custom-validators/misc'
import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'
import { VideoChannelModel } from '../../../models/video/video-channel'
import * as Bluebird from 'bluebird'
const auditLogger = auditLoggerFactory('video-imports')
const videoImportsRouter = express.Router()
@ -41,7 +46,45 @@ export {
// ---------------------------------------------------------------------------
async function addVideoImport (req: express.Request, res: express.Response) {
function addVideoImport (req: express.Request, res: express.Response) {
if (req.body.targetUrl) return addYoutubeDLImport(req, res)
if (req.body.magnetUri) return addTorrentImport(req, res)
}
async function addTorrentImport (req: express.Request, res: express.Response) {
const body: VideoImportCreate = req.body
const magnetUri = body.magnetUri
const parsed = magnetUtil.decode(magnetUri)
const magnetName = isArray(parsed.name) ? parsed.name[0] : parsed.name as string
const video = buildVideo(res.locals.videoChannel.id, body, { name: magnetName })
await processThumbnail(req, video)
await processPreview(req, video)
const tags = null
const videoImportAttributes = {
magnetUri,
state: VideoImportState.PENDING
}
const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes)
// Create job to import the video
const payload = {
type: 'magnet-uri' as 'magnet-uri',
videoImportId: videoImport.id,
magnetUri
}
await JobQueue.Instance.createJob({ type: 'video-import', payload })
auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON()))
return res.json(videoImport.toFormattedJSON()).end()
}
async function addYoutubeDLImport (req: express.Request, res: express.Response) {
const body: VideoImportCreate = req.body
const targetUrl = body.targetUrl
@ -56,71 +99,17 @@ async function addVideoImport (req: express.Request, res: express.Response) {
}).end()
}
// Create video DB object
const videoData = {
name: body.name || youtubeDLInfo.name,
remote: false,
category: body.category || youtubeDLInfo.category,
licence: body.licence || youtubeDLInfo.licence,
language: body.language || undefined,
commentsEnabled: body.commentsEnabled || true,
waitTranscoding: body.waitTranscoding || false,
state: VideoState.TO_IMPORT,
nsfw: body.nsfw || youtubeDLInfo.nsfw || false,
description: body.description || youtubeDLInfo.description,
support: body.support || null,
privacy: body.privacy || VideoPrivacy.PRIVATE,
duration: 0, // duration will be set by the import job
channelId: res.locals.videoChannel.id
const video = buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
const downloadThumbnail = !await processThumbnail(req, video)
const downloadPreview = !await processPreview(req, video)
const tags = body.tags || youtubeDLInfo.tags
const videoImportAttributes = {
targetUrl,
state: VideoImportState.PENDING
}
const video = new VideoModel(videoData)
video.url = getVideoActivityPubUrl(video)
// Process thumbnail file?
const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined
let downloadThumbnail = true
if (thumbnailField) {
const thumbnailPhysicalFile = thumbnailField[ 0 ]
await processImage(thumbnailPhysicalFile, join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()), THUMBNAILS_SIZE)
downloadThumbnail = false
}
// Process preview file?
const previewField = req.files ? req.files['previewfile'] : undefined
let downloadPreview = true
if (previewField) {
const previewPhysicalFile = previewField[0]
await processImage(previewPhysicalFile, join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()), PREVIEWS_SIZE)
downloadPreview = false
}
const videoImport: VideoImportModel = await sequelizeTypescript.transaction(async t => {
const sequelizeOptions = { transaction: t }
// Save video object in database
const videoCreated = await video.save(sequelizeOptions)
videoCreated.VideoChannel = res.locals.videoChannel
// Set tags to the video
const tags = body.tags ? body.tags : youtubeDLInfo.tags
if (tags !== undefined) {
const tagInstances = await TagModel.findOrCreateTags(tags, t)
await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
videoCreated.Tags = tagInstances
}
// Create video import object in database
const videoImport = await VideoImportModel.create({
targetUrl,
state: VideoImportState.PENDING,
videoId: videoCreated.id
}, sequelizeOptions)
videoImport.Video = videoCreated
return videoImport
})
const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes)
// Create job to import the video
const payload = {
@ -136,3 +125,82 @@ async function addVideoImport (req: express.Request, res: express.Response) {
return res.json(videoImport.toFormattedJSON()).end()
}
function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo) {
const videoData = {
name: body.name || importData.name || 'Unknown name',
remote: false,
category: body.category || importData.category,
licence: body.licence || importData.licence,
language: body.language || undefined,
commentsEnabled: body.commentsEnabled || true,
waitTranscoding: body.waitTranscoding || false,
state: VideoState.TO_IMPORT,
nsfw: body.nsfw || importData.nsfw || false,
description: body.description || importData.description,
support: body.support || null,
privacy: body.privacy || VideoPrivacy.PRIVATE,
duration: 0, // duration will be set by the import job
channelId: channelId
}
const video = new VideoModel(videoData)
video.url = getVideoActivityPubUrl(video)
return video
}
async function processThumbnail (req: express.Request, video: VideoModel) {
const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined
if (thumbnailField) {
const thumbnailPhysicalFile = thumbnailField[ 0 ]
await processImage(thumbnailPhysicalFile, join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()), THUMBNAILS_SIZE)
return true
}
return false
}
async function processPreview (req: express.Request, video: VideoModel) {
const previewField = req.files ? req.files['previewfile'] : undefined
if (previewField) {
const previewPhysicalFile = previewField[0]
await processImage(previewPhysicalFile, join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()), PREVIEWS_SIZE)
return true
}
return false
}
function insertIntoDB (
video: VideoModel,
videoChannel: VideoChannelModel,
tags: string[],
videoImportAttributes: FilteredModelAttributes<VideoImportModel>
): Bluebird<VideoImportModel> {
return sequelizeTypescript.transaction(async t => {
const sequelizeOptions = { transaction: t }
// Save video object in database
const videoCreated = await video.save(sequelizeOptions)
videoCreated.VideoChannel = videoChannel
// Set tags to the video
if (tags !== undefined) {
const tagInstances = await TagModel.findOrCreateTags(tags, t)
await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
videoCreated.Tags = tagInstances
}
// Create video import object in database
const videoImport = await VideoImportModel.create(
Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
sequelizeOptions
)
videoImport.Video = videoCreated
return videoImport
})
}

View File

@ -17,6 +17,7 @@ import { VideoModel } from '../../models/video/video'
import { exists, isArray, isFileValid } from './misc'
import { VideoChannelModel } from '../../models/video/video-channel'
import { UserModel } from '../../models/account/user'
import * as magnetUtil from 'magnet-uri'
const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS
const VIDEO_ABUSES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_ABUSES
@ -126,6 +127,13 @@ function isVideoFileSizeValid (value: string) {
return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.FILE_SIZE)
}
function isVideoMagnetUriValid (value: string) {
if (!exists(value)) return false
const parsed = magnetUtil.decode(value)
return parsed && isVideoFileInfoHashValid(parsed.infoHash)
}
function checkUserCanManageVideo (user: UserModel, video: VideoModel, right: UserRight, res: Response) {
// Retrieve the user who did the request
if (video.isOwned() === false) {
@ -214,6 +222,7 @@ export {
isScheduleVideoUpdatePrivacyValid,
isVideoAbuseReasonValid,
isVideoFile,
isVideoMagnetUriValid,
isVideoStateValid,
isVideoViewsValid,
isVideoRatingTypeValid,

View File

@ -9,6 +9,8 @@ import { ApplicationModel } from '../models/application/application'
import { pseudoRandomBytesPromise, unlinkPromise } from './core-utils'
import { logger } from './logger'
import { isArray } from './custom-validators/misc'
import * as crypto from "crypto"
import { join } from "path"
const isCidr = require('is-cidr')
@ -181,8 +183,14 @@ async function getServerActor () {
return Promise.resolve(serverActor)
}
function generateVideoTmpPath (id: string) {
const hash = crypto.createHash('sha256').update(id).digest('hex')
return join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4')
}
type SortType = { sortModel: any, sortValue: string }
// ---------------------------------------------------------------------------
export {
@ -195,5 +203,6 @@ export {
computeResolutionsToTranscode,
resetSequelizeInstance,
getServerActor,
SortType
SortType,
generateVideoTmpPath
}

View File

@ -0,0 +1,31 @@
import { logger } from './logger'
import { generateVideoTmpPath } from './utils'
import * as WebTorrent from 'webtorrent'
import { createWriteStream } from 'fs'
function downloadWebTorrentVideo (target: string) {
const path = generateVideoTmpPath(target)
logger.info('Importing torrent video %s', target)
return new Promise<string>((res, rej) => {
const webtorrent = new WebTorrent()
const torrent = webtorrent.add(target, torrent => {
if (torrent.files.length !== 1) throw new Error('The number of files is not equal to 1 for ' + target)
const file = torrent.files[ 0 ]
file.createReadStream().pipe(createWriteStream(path))
})
torrent.on('done', () => res(path))
torrent.on('error', err => rej(err))
})
}
// ---------------------------------------------------------------------------
export {
downloadWebTorrentVideo
}

View File

@ -1,18 +1,17 @@
import * as youtubeDL from 'youtube-dl'
import { truncate } from 'lodash'
import { CONFIG, CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers'
import { join } from 'path'
import * as crypto from 'crypto'
import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers'
import { logger } from './logger'
import { generateVideoTmpPath } from './utils'
export type YoutubeDLInfo = {
name: string
description: string
category: number
licence: number
nsfw: boolean
tags: string[]
thumbnailUrl: string
name?: string
description?: string
category?: number
licence?: number
nsfw?: boolean
tags?: string[]
thumbnailUrl?: string
}
function getYoutubeDLInfo (url: string): Promise<YoutubeDLInfo> {
@ -30,10 +29,9 @@ function getYoutubeDLInfo (url: string): Promise<YoutubeDLInfo> {
}
function downloadYoutubeDLVideo (url: string) {
const hash = crypto.createHash('sha256').update(url).digest('hex')
const path = join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4')
const path = generateVideoTmpPath(url)
logger.info('Importing video %s', url)
logger.info('Importing youtubeDL video %s', url)
const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ]

View File

@ -15,7 +15,7 @@ let config: IConfig = require('config')
// ---------------------------------------------------------------------------
const LAST_MIGRATION_VERSION = 240
const LAST_MIGRATION_VERSION = 245
// ---------------------------------------------------------------------------
@ -271,7 +271,8 @@ const CONSTRAINTS_FIELDS = {
}
},
VIDEO_IMPORTS: {
URL: { min: 3, max: 2000 } // Length
URL: { min: 3, max: 2000 }, // Length
TORRENT_NAME: { min: 3, max: 255 }, // Length
},
VIDEOS: {
NAME: { min: 3, max: 120 }, // Length

View File

@ -0,0 +1,42 @@
import * as Sequelize from 'sequelize'
import { Migration } from '../../models/migrations'
import { CONSTRAINTS_FIELDS } from '../index'
async function up (utils: {
transaction: Sequelize.Transaction
queryInterface: Sequelize.QueryInterface
sequelize: Sequelize.Sequelize
}): Promise<any> {
{
const data = {
type: Sequelize.STRING,
allowNull: true,
defaultValue: null
} as Migration.String
await utils.queryInterface.changeColumn('videoImport', 'targetUrl', data)
}
{
const data = {
type: Sequelize.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max),
allowNull: true,
defaultValue: null
}
await utils.queryInterface.addColumn('videoImport', 'magnetUri', data)
}
{
const data = {
type: Sequelize.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_NAME.max),
allowNull: true,
defaultValue: null
}
await utils.queryInterface.addColumn('videoImport', 'torrentName', data)
}
}
function down (options) {
throw new Error('Not implemented.')
}
export { up, down }

View File

@ -13,30 +13,99 @@ import { VideoState } from '../../../../shared'
import { JobQueue } from '../index'
import { federateVideoIfNeeded } from '../../activitypub'
import { VideoModel } from '../../../models/video/video'
import { downloadWebTorrentVideo } from '../../../helpers/webtorrent'
export type VideoImportPayload = {
type VideoImportYoutubeDLPayload = {
type: 'youtube-dl'
videoImportId: number
thumbnailUrl: string
downloadThumbnail: boolean
downloadPreview: boolean
}
type VideoImportTorrentPayload = {
type: 'magnet-uri'
videoImportId: number
}
export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload
async function processVideoImport (job: Bull.Job) {
const payload = job.data as VideoImportPayload
logger.info('Processing video import in job %d.', job.id)
const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId)
if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, payload)
if (payload.type === 'magnet-uri') return processTorrentImport(job, payload)
}
// ---------------------------------------------------------------------------
export {
processVideoImport
}
// ---------------------------------------------------------------------------
async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentPayload) {
logger.info('Processing torrent video import in job %d.', job.id)
const videoImport = await getVideoImportOrDie(payload.videoImportId)
const options = {
videoImportId: payload.videoImportId,
downloadThumbnail: false,
downloadPreview: false,
generateThumbnail: true,
generatePreview: true
}
return processFile(() => downloadWebTorrentVideo(videoImport.magnetUri), videoImport, options)
}
async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutubeDLPayload) {
logger.info('Processing youtubeDL video import in job %d.', job.id)
const videoImport = await getVideoImportOrDie(payload.videoImportId)
const options = {
videoImportId: videoImport.id,
downloadThumbnail: payload.downloadThumbnail,
downloadPreview: payload.downloadPreview,
thumbnailUrl: payload.thumbnailUrl,
generateThumbnail: false,
generatePreview: false
}
return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl), videoImport, options)
}
async function getVideoImportOrDie (videoImportId: number) {
const videoImport = await VideoImportModel.loadAndPopulateVideo(videoImportId)
if (!videoImport || !videoImport.Video) {
throw new Error('Cannot import video %s: the video import or video linked to this import does not exist anymore.')
}
return videoImport
}
type ProcessFileOptions = {
videoImportId: number
downloadThumbnail: boolean
downloadPreview: boolean
thumbnailUrl?: string
generateThumbnail: boolean
generatePreview: boolean
}
async function processFile (downloader: () => Promise<string>, videoImport: VideoImportModel, options: ProcessFileOptions) {
let tempVideoPath: string
let videoDestFile: string
let videoFile: VideoFileModel
try {
// Download video from youtubeDL
tempVideoPath = await downloadYoutubeDLVideo(videoImport.targetUrl)
tempVideoPath = await downloader()
// Get information about this video
const { videoFileResolution } = await getVideoFileResolution(tempVideoPath)
@ -62,23 +131,27 @@ async function processVideoImport (job: Bull.Job) {
tempVideoPath = null // This path is not used anymore
// Process thumbnail
if (payload.downloadThumbnail) {
if (payload.thumbnailUrl) {
if (options.downloadThumbnail) {
if (options.thumbnailUrl) {
const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName())
await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destThumbnailPath)
await doRequestAndSaveToFile({ method: 'GET', uri: options.thumbnailUrl }, destThumbnailPath)
} else {
await videoImport.Video.createThumbnail(videoFile)
}
} else if (options.generateThumbnail) {
await videoImport.Video.createThumbnail(videoFile)
}
// Process preview
if (payload.downloadPreview) {
if (payload.thumbnailUrl) {
if (options.downloadPreview) {
if (options.thumbnailUrl) {
const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName())
await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destPreviewPath)
await doRequestAndSaveToFile({ method: 'GET', uri: options.thumbnailUrl }, destPreviewPath)
} else {
await videoImport.Video.createPreview(videoFile)
}
} else if (options.generatePreview) {
await videoImport.Video.createPreview(videoFile)
}
// Create torrent
@ -137,9 +210,3 @@ async function processVideoImport (job: Bull.Job) {
throw err
}
}
// ---------------------------------------------------------------------------
export {
processVideoImport
}

View File

@ -32,13 +32,6 @@ const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
'video-import': processVideoImport
}
const jobsWithRequestTimeout: { [ id in JobType ]?: boolean } = {
'activitypub-http-broadcast': true,
'activitypub-http-unicast': true,
'activitypub-http-fetcher': true,
'activitypub-follow': true
}
const jobTypes: JobType[] = [
'activitypub-follow',
'activitypub-http-broadcast',

View File

@ -6,14 +6,19 @@ import { areValidationErrors } from './utils'
import { getCommonVideoAttributes } from './videos'
import { isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports'
import { cleanUpReqFiles } from '../../helpers/utils'
import { isVideoChannelOfAccountExist, isVideoNameValid } from '../../helpers/custom-validators/videos'
import { isVideoChannelOfAccountExist, isVideoMagnetUriValid, isVideoNameValid } from '../../helpers/custom-validators/videos'
import { CONFIG } from '../../initializers/constants'
const videoImportAddValidator = getCommonVideoAttributes().concat([
body('targetUrl').custom(isVideoImportTargetUrlValid).withMessage('Should have a valid video import target URL'),
body('channelId')
.toInt()
.custom(isIdValid).withMessage('Should have correct video channel id'),
body('targetUrl')
.optional()
.custom(isVideoImportTargetUrlValid).withMessage('Should have a valid video import target URL'),
body('magnetUri')
.optional()
.custom(isVideoMagnetUriValid).withMessage('Should have a valid video magnet URI'),
body('name')
.optional()
.custom(isVideoNameValid).withMessage('Should have a valid name'),
@ -34,6 +39,15 @@ const videoImportAddValidator = getCommonVideoAttributes().concat([
if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req)
// Check we have at least 1 required param
if (!req.body.targetUrl && !req.body.magnetUri) {
cleanUpReqFiles(req)
return res.status(400)
.json({ error: 'Should have a magnetUri or a targetUrl.' })
.end()
}
return next()
}
])

View File

@ -21,6 +21,7 @@ import { VideoImport, VideoImportState } from '../../../shared'
import { VideoChannelModel } from './video-channel'
import { AccountModel } from '../account/account'
import { TagModel } from './tag'
import { isVideoMagnetUriValid } from '../../helpers/custom-validators/videos'
@DefaultScope({
include: [
@ -62,11 +63,23 @@ export class VideoImportModel extends Model<VideoImportModel> {
@UpdatedAt
updatedAt: Date
@AllowNull(false)
@AllowNull(true)
@Default(null)
@Is('VideoImportTargetUrl', value => throwIfNotValid(value, isVideoImportTargetUrlValid, 'targetUrl'))
@Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max))
targetUrl: string
@AllowNull(true)
@Default(null)
@Is('VideoImportMagnetUri', value => throwIfNotValid(value, isVideoMagnetUriValid, 'magnetUri'))
@Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max)) // Use the same constraints than URLs
magnetUri: string
@AllowNull(true)
@Default(null)
@Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_NAME.max))
torrentName: string
@AllowNull(false)
@Default(null)
@Is('VideoImportState', value => throwIfNotValid(value, isVideoImportStateValid, 'state'))

View File

@ -1,6 +1,9 @@
import { VideoUpdate } from './video-update.model'
export interface VideoImportCreate extends VideoUpdate {
targetUrl: string
targetUrl?: string
magnetUri?: string
torrentfile?: Blob
channelId: number // Required
}