PeerTube/server/models/video.ts

import * as safeBuffer from 'safe-buffer'
const Buffer = safeBuffer.Buffer
import * as createTorrent from 'create-torrent'
import * as ffmpeg from 'fluent-ffmpeg'
import * as fs from 'fs'
import * as magnetUtil from 'magnet-uri'
import { map, values } from 'lodash'
import { parallel, series } from 'async'
import * as parseTorrent from 'parse-torrent'
import { join } from 'path'
import * as Sequelize from 'sequelize'

import { database as db } from '../initializers/database'
import {
  logger,
  isVideoNameValid,
  isVideoCategoryValid,
  isVideoLicenceValid,
  isVideoLanguageValid,
  isVideoNSFWValid,
  isVideoDescriptionValid,
  isVideoInfoHashValid,
  isVideoDurationValid
} from '../helpers'
import {
  CONSTRAINTS_FIELDS,
  CONFIG,
  REMOTE_SCHEME,
  STATIC_PATHS,
  VIDEO_CATEGORIES,
  VIDEO_LICENCES,
  VIDEO_LANGUAGES,
  THUMBNAILS_SIZE
} from '../initializers'
import { JobScheduler, removeVideoToFriends } from '../lib'

import { addMethodsToModel, getSort } from './utils'
import {
  VideoClass,
  VideoInstance,
  VideoAttributes,

  VideoMethods
} from './video-interface'

let Video: Sequelize.Model<VideoInstance, VideoAttributes>
let generateMagnetUri: VideoMethods.GenerateMagnetUri
let getVideoFilename: VideoMethods.GetVideoFilename
let getThumbnailName: VideoMethods.GetThumbnailName
let getPreviewName: VideoMethods.GetPreviewName
let getTorrentName: VideoMethods.GetTorrentName
let isOwned: VideoMethods.IsOwned
let toFormatedJSON: VideoMethods.ToFormatedJSON
let toAddRemoteJSON: VideoMethods.ToAddRemoteJSON
let toUpdateRemoteJSON: VideoMethods.ToUpdateRemoteJSON
let transcodeVideofile: VideoMethods.TranscodeVideofile

let generateThumbnailFromData: VideoMethods.GenerateThumbnailFromData
let getDurationFromFile: VideoMethods.GetDurationFromFile
let list: VideoMethods.List
let listForApi: VideoMethods.ListForApi
let loadByHostAndRemoteId: VideoMethods.LoadByHostAndRemoteId
let listOwnedAndPopulateAuthorAndTags: VideoMethods.ListOwnedAndPopulateAuthorAndTags
let listOwnedByAuthor: VideoMethods.ListOwnedByAuthor
let load: VideoMethods.Load
let loadAndPopulateAuthor: VideoMethods.LoadAndPopulateAuthor
let loadAndPopulateAuthorAndPodAndTags: VideoMethods.LoadAndPopulateAuthorAndPodAndTags
let searchAndPopulateAuthorAndPodAndTags: VideoMethods.SearchAndPopulateAuthorAndPodAndTags

export default function (sequelize, DataTypes) {
  Video = sequelize.define('Video',
    {
      id: {
        type: DataTypes.UUID,
        defaultValue: DataTypes.UUIDV4,
        primaryKey: true,
        validate: {
          isUUID: 4
        }
      },
      name: {
        type: DataTypes.STRING,
        allowNull: false,
        validate: {
          nameValid: function (value) {
            const res = isVideoNameValid(value)
            if (res === false) throw new Error('Video name is not valid.')
          }
        }
      },
      extname: {
        type: DataTypes.ENUM(values(CONSTRAINTS_FIELDS.VIDEOS.EXTNAME)),
        allowNull: false
      },
      remoteId: {
        type: DataTypes.UUID,
        allowNull: true,
        validate: {
          isUUID: 4
        }
      },
      category: {
        type: DataTypes.INTEGER,
        allowNull: false,
        validate: {
          categoryValid: function (value) {
            const res = isVideoCategoryValid(value)
            if (res === false) throw new Error('Video category is not valid.')
          }
        }
      },
      licence: {
        type: DataTypes.INTEGER,
        allowNull: false,
        defaultValue: null,
        validate: {
          licenceValid: function (value) {
            const res = isVideoLicenceValid(value)
            if (res === false) throw new Error('Video licence is not valid.')
          }
        }
      },
      language: {
        type: DataTypes.INTEGER,
        allowNull: true,
        validate: {
          languageValid: function (value) {
            const res = isVideoLanguageValid(value)
            if (res === false) throw new Error('Video language is not valid.')
          }
        }
      },
      nsfw: {
        type: DataTypes.BOOLEAN,
        allowNull: false,
        validate: {
          nsfwValid: function (value) {
            const res = isVideoNSFWValid(value)
            if (res === false) throw new Error('Video nsfw attribute is not valid.')
          }
        }
      },
      description: {
        type: DataTypes.STRING,
        allowNull: false,
        validate: {
          descriptionValid: function (value) {
            const res = isVideoDescriptionValid(value)
            if (res === false) throw new Error('Video description is not valid.')
          }
        }
      },
      infoHash: {
        type: DataTypes.STRING,
        allowNull: false,
        validate: {
          infoHashValid: function (value) {
            const res = isVideoInfoHashValid(value)
            if (res === false) throw new Error('Video info hash is not valid.')
          }
        }
      },
      duration: {
        type: DataTypes.INTEGER,
        allowNull: false,
        validate: {
          durationValid: function (value) {
            const res = isVideoDurationValid(value)
            if (res === false) throw new Error('Video duration is not valid.')
          }
        }
      },
      views: {
        type: DataTypes.INTEGER,
        allowNull: false,
        defaultValue: 0,
        validate: {
          min: 0,
          isInt: true
        }
      },
      likes: {
        type: DataTypes.INTEGER,
        allowNull: false,
        defaultValue: 0,
        validate: {
          min: 0,
          isInt: true
        }
      },
      dislikes: {
        type: DataTypes.INTEGER,
        allowNull: false,
        defaultValue: 0,
        validate: {
          min: 0,
          isInt: true
        }
      }
    },
    {
      indexes: [
        {
          fields: [ 'authorId' ]
        },
        {
          fields: [ 'remoteId' ]
        },
        {
          fields: [ 'name' ]
        },
        {
          fields: [ 'createdAt' ]
        },
        {
          fields: [ 'duration' ]
        },
        {
          fields: [ 'infoHash' ]
        },
        {
          fields: [ 'views' ]
        },
        {
          fields: [ 'likes' ]
        }
      ],
      hooks: {
        beforeValidate,
        beforeCreate,
        afterDestroy
      }
    }
  )

  const classMethods = [
    associate,

    generateThumbnailFromData,
    getDurationFromFile,
    list,
    listForApi,
    listOwnedAndPopulateAuthorAndTags,
    listOwnedByAuthor,
    load,
    loadByHostAndRemoteId,
    loadAndPopulateAuthor,
    loadAndPopulateAuthorAndPodAndTags,
    searchAndPopulateAuthorAndPodAndTags
  ]
  const instanceMethods = [
    generateMagnetUri,
    getVideoFilename,
    getThumbnailName,
    getPreviewName,
    getTorrentName,
    isOwned,
    toFormatedJSON,
    toAddRemoteJSON,
    toUpdateRemoteJSON,
    transcodeVideofile,
    removeFromBlacklist
  ]
  addMethodsToModel(Video, classMethods, instanceMethods)

  return Video
}

function beforeValidate (video, options) {
  // Put a fake infoHash if it does not exists yet
  if (video.isOwned() && !video.infoHash) {
    // 40 hexa length
    video.infoHash = '0123456789abcdef0123456789abcdef01234567'
  }
}

function beforeCreate (video, options) {
  return new Promise(function (resolve, reject) {
    const tasks = []

    if (video.isOwned()) {
      const videoPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename())

      tasks.push(
        function createVideoTorrent (callback) {
          createTorrentFromVideo(video, videoPath, callback)
        },

        function createVideoThumbnail (callback) {
          createThumbnail(video, videoPath, callback)
        },

        function createVideoPreview (callback) {
          createPreview(video, videoPath, callback)
        }
      )

      if (CONFIG.TRANSCODING.ENABLED === true) {
        tasks.push(
          function createVideoTranscoderJob (callback) {
            const dataInput = {
              id: video.id
            }

            JobScheduler.Instance.createJob(options.transaction, 'videoTranscoder', dataInput, callback)
          }
        )
      }

      return parallel(tasks, function (err) {
        if (err) return reject(err)

        return resolve()
      })
    }

    return resolve()
  })
}

function afterDestroy (video, options) {
  return new Promise(function (resolve, reject) {
    const tasks = []

    tasks.push(
      function (callback) {
        removeThumbnail(video, callback)
      }
    )

    if (video.isOwned()) {
      tasks.push(
        function removeVideoFile (callback) {
          removeFile(video, callback)
        },

        function removeVideoTorrent (callback) {
          removeTorrent(video, callback)
        },

        function removeVideoPreview (callback) {
          removePreview(video, callback)
        },

        function notifyFriends (callback) {
          const params = {
            remoteId: video.id
          }

          removeVideoToFriends(params)

          return callback()
        }
      )
    }

    parallel(tasks, function (err) {
      if (err) return reject(err)

      return resolve()
    })
  })
}

// ------------------------------ METHODS ------------------------------

function associate (models) {
  Video.belongsTo(models.Author, {
    foreignKey: {
      name: 'authorId',
      allowNull: false
    },
    onDelete: 'cascade'
  })

  Video.belongsToMany(models.Tag, {
    foreignKey: 'videoId',
    through: models.VideoTag,
    onDelete: 'cascade'
  })

  Video.hasMany(models.VideoAbuse, {
    foreignKey: {
      name: 'videoId',
      allowNull: false
    },
    onDelete: 'cascade'
  })
}

generateMagnetUri = function () {
  let baseUrlHttp
  let baseUrlWs

  if (this.isOwned()) {
    baseUrlHttp = CONFIG.WEBSERVER.URL
    baseUrlWs = CONFIG.WEBSERVER.WS + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT
  } else {
    baseUrlHttp = REMOTE_SCHEME.HTTP + '://' + this.Author.Pod.host
    baseUrlWs = REMOTE_SCHEME.WS + '://' + this.Author.Pod.host
  }

  const xs = baseUrlHttp + STATIC_PATHS.TORRENTS + this.getTorrentName()
  const announce = baseUrlWs + '/tracker/socket'
  const urlList = [ baseUrlHttp + STATIC_PATHS.WEBSEED + this.getVideoFilename() ]

  const magnetHash = {
    xs,
    announce,
    urlList,
    infoHash: this.infoHash,
    name: this.name
  }

  return magnetUtil.encode(magnetHash)
}

getVideoFilename = function () {
  if (this.isOwned()) return this.id + this.extname

  return this.remoteId + this.extname
}

getThumbnailName = function () {
  // We always have a copy of the thumbnail
  return this.id + '.jpg'
}

getPreviewName = function () {
  const extension = '.jpg'

  if (this.isOwned()) return this.id + extension

  return this.remoteId + extension
}

getTorrentName = function () {
  const extension = '.torrent'

  if (this.isOwned()) return this.id + extension

  return this.remoteId + extension
}

isOwned = function () {
  return this.remoteId === null
}

toFormatedJSON = function () {
  let podHost

  if (this.Author.Pod) {
    podHost = this.Author.Pod.host
  } else {
    // It means it's our video
    podHost = CONFIG.WEBSERVER.HOST
  }

  // Maybe our pod is not up to date and there are new categories since our version
  let categoryLabel = VIDEO_CATEGORIES[this.category]
  if (!categoryLabel) categoryLabel = 'Misc'

  // Maybe our pod is not up to date and there are new licences since our version
  let licenceLabel = VIDEO_LICENCES[this.licence]
  if (!licenceLabel) licenceLabel = 'Unknown'

  // Language is an optional attribute
  let languageLabel = VIDEO_LANGUAGES[this.language]
  if (!languageLabel) languageLabel = 'Unknown'

  const json = {
    id: this.id,
    name: this.name,
    category: this.category,
    categoryLabel,
    licence: this.licence,
    licenceLabel,
    language: this.language,
    languageLabel,
    nsfw: this.nsfw,
    description: this.description,
    podHost,
    isLocal: this.isOwned(),
    magnetUri: this.generateMagnetUri(),
    author: this.Author.name,
    duration: this.duration,
    views: this.views,
    likes: this.likes,
    dislikes: this.dislikes,
    tags: map(this.Tags, 'name'),
    thumbnailPath: join(STATIC_PATHS.THUMBNAILS, this.getThumbnailName()),
    createdAt: this.createdAt,
    updatedAt: this.updatedAt
  }

  return json
}

toAddRemoteJSON = function (callback) {
  // Get thumbnail data to send to the other pod
  const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, this.getThumbnailName())
  fs.readFile(thumbnailPath, (err, thumbnailData) => {
    if (err) {
      logger.error('Cannot read the thumbnail of the video')
      return callback(err)
    }

    const remoteVideo = {
      name: this.name,
      category: this.category,
      licence: this.licence,
      language: this.language,
      nsfw: this.nsfw,
      description: this.description,
      infoHash: this.infoHash,
      remoteId: this.id,
      author: this.Author.name,
      duration: this.duration,
      thumbnailData: thumbnailData.toString('binary'),
      tags: map(this.Tags, 'name'),
      createdAt: this.createdAt,
      updatedAt: this.updatedAt,
      extname: this.extname,
      views: this.views,
      likes: this.likes,
      dislikes: this.dislikes
    }

    return callback(null, remoteVideo)
  })
}

toUpdateRemoteJSON = function (callback) {
  const json = {
    name: this.name,
    category: this.category,
    licence: this.licence,
    language: this.language,
    nsfw: this.nsfw,
    description: this.description,
    infoHash: this.infoHash,
    remoteId: this.id,
    author: this.Author.name,
    duration: this.duration,
    tags: map(this.Tags, 'name'),
    createdAt: this.createdAt,
    updatedAt: this.updatedAt,
    extname: this.extname,
    views: this.views,
    likes: this.likes,
    dislikes: this.dislikes
  }

  return json
}

transcodeVideofile = function (finalCallback) {
  const video = this

  const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
  const newExtname = '.mp4'
  const videoInputPath = join(videosDirectory, video.getVideoFilename())
  const videoOutputPath = join(videosDirectory, video.id + '-transcoded' + newExtname)

  ffmpeg(videoInputPath)
    .output(videoOutputPath)
    .videoCodec('libx264')
    .outputOption('-threads ' + CONFIG.TRANSCODING.THREADS)
    .outputOption('-movflags faststart')
    .on('error', finalCallback)
    .on('end', function () {
      series([
        function removeOldFile (callback) {
          fs.unlink(videoInputPath, callback)
        },

        function moveNewFile (callback) {
          // Important to do this before getVideoFilename() to take in account the new file extension
          video.set('extname', newExtname)

          const newVideoPath = join(videosDirectory, video.getVideoFilename())
          fs.rename(videoOutputPath, newVideoPath, callback)
        },

        function torrent (callback) {
          const newVideoPath = join(videosDirectory, video.getVideoFilename())
          createTorrentFromVideo(video, newVideoPath, callback)
        },

        function videoExtension (callback) {
          video.save().asCallback(callback)
        }

      ], function (err) {
        if (err) {
          // Autodescruction...
          video.destroy().asCallback(function (err) {
            if (err) logger.error('Cannot destruct video after transcoding failure.', { error: err })
          })

          return finalCallback(err)
        }

        return finalCallback(null)
      })
    })
    .run()
}

// ------------------------------ STATICS ------------------------------

generateThumbnailFromData = function (video, thumbnailData, callback) {
  // Creating the thumbnail for a remote video

  const thumbnailName = video.getThumbnailName()
  const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName)
  fs.writeFile(thumbnailPath, Buffer.from(thumbnailData, 'binary'), function (err) {
    if (err) return callback(err)

    return callback(null, thumbnailName)
  })
}

getDurationFromFile = function (videoPath, callback) {
  ffmpeg.ffprobe(videoPath, function (err, metadata) {
    if (err) return callback(err)

    return callback(null, Math.floor(metadata.format.duration))
  })
}

list = function (callback) {
  return Video.findAll().asCallback(callback)
}

listForApi = function (start, count, sort, callback) {
  // Exclude Blakclisted videos from the list
  const query = {
    distinct: true,
    offset: start,
    limit: count,
    order: [ getSort(sort), [ Video['sequelize'].models.Tag, 'name', 'ASC' ] ],
    include: [
      {
        model: Video['sequelize'].models.Author,
        include: [ { model: Video['sequelize'].models.Pod, required: false } ]
      },

      Video['sequelize'].models.Tag
    ],
    where: createBaseVideosWhere()
  }

  return Video.findAndCountAll(query).asCallback(function (err, result) {
    if (err) return callback(err)

    return callback(null, result.rows, result.count)
  })
}

loadByHostAndRemoteId = function (fromHost, remoteId, callback) {
  const query = {
    where: {
      remoteId: remoteId
    },
    include: [
      {
        model: Video['sequelize'].models.Author,
        include: [
          {
            model: Video['sequelize'].models.Pod,
            required: true,
            where: {
              host: fromHost
            }
          }
        ]
      }
    ]
  }

  return Video.findOne(query).asCallback(callback)
}

listOwnedAndPopulateAuthorAndTags = function (callback) {
  // If remoteId is null this is *our* video
  const query = {
    where: {
      remoteId: null
    },
    include: [ Video['sequelize'].models.Author, Video['sequelize'].models.Tag ]
  }

  return Video.findAll(query).asCallback(callback)
}

listOwnedByAuthor = function (author, callback) {
  const query = {
    where: {
      remoteId: null
    },
    include: [
      {
        model: Video['sequelize'].models.Author,
        where: {
          name: author
        }
      }
    ]
  }

  return Video.findAll(query).asCallback(callback)
}

load = function (id, callback) {
  return Video.findById(id).asCallback(callback)
}

loadAndPopulateAuthor = function (id, callback) {
  const options = {
    include: [ Video['sequelize'].models.Author ]
  }

  return Video.findById(id, options).asCallback(callback)
}

loadAndPopulateAuthorAndPodAndTags = function (id, callback) {
  const options = {
    include: [
      {
        model: Video['sequelize'].models.Author,
        include: [ { model: Video['sequelize'].models.Pod, required: false } ]
      },
      Video['sequelize'].models.Tag
    ]
  }

  return Video.findById(id, options).asCallback(callback)
}

searchAndPopulateAuthorAndPodAndTags = function (value, field, start, count, sort, callback) {
  const podInclude: any = {
    model: Video['sequelize'].models.Pod,
    required: false
  }

  const authorInclude: any = {
    model: Video['sequelize'].models.Author,
    include: [
      podInclude
    ]
  }

  const tagInclude: any = {
    model: Video['sequelize'].models.Tag
  }

  const query: any = {
    distinct: true,
    where: createBaseVideosWhere(),
    offset: start,
    limit: count,
    order: [ getSort(sort), [ Video['sequelize'].models.Tag, 'name', 'ASC' ] ]
  }

  // Make an exact search with the magnet
  if (field === 'magnetUri') {
    const infoHash = magnetUtil.decode(value).infoHash
    query.where.infoHash = infoHash
  } else if (field === 'tags') {
    const escapedValue = Video['sequelize'].escape('%' + value + '%')
    query.where.id.$in = Video['sequelize'].literal(
      '(SELECT "VideoTags"."videoId" FROM "Tags" INNER JOIN "VideoTags" ON "Tags"."id" = "VideoTags"."tagId" WHERE name LIKE ' + escapedValue + ')'
    )
  } else if (field === 'host') {
    // FIXME: Include our pod? (not stored in the database)
    podInclude.where = {
      host: {
        $like: '%' + value + '%'
      }
    }
    podInclude.required = true
  } else if (field === 'author') {
    authorInclude.where = {
      name: {
        $like: '%' + value + '%'
      }
    }

    // authorInclude.or = true
  } else {
    query.where[field] = {
      $like: '%' + value + '%'
    }
  }

  query.include = [
    authorInclude, tagInclude
  ]

  if (tagInclude.where) {
    // query.include.push([ Video['sequelize'].models.Tag ])
  }

  return Video.findAndCountAll(query).asCallback(function (err, result) {
    if (err) return callback(err)

    return callback(null, result.rows, result.count)
  })
}

// ---------------------------------------------------------------------------

function createBaseVideosWhere () {
  return {
    id: {
      $notIn: Video['sequelize'].literal(
        '(SELECT "BlacklistedVideos"."videoId" FROM "BlacklistedVideos")'
      )
    }
  }
}

function removeThumbnail (video, callback) {
  const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName())
  fs.unlink(thumbnailPath, callback)
}

function removeFile (video, callback) {
  const filePath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename())
  fs.unlink(filePath, callback)
}

function removeTorrent (video, callback) {
  const torrenPath = join(CONFIG.STORAGE.TORRENTS_DIR, video.getTorrentName())
  fs.unlink(torrenPath, callback)
}

function removePreview (video, callback) {
  // Same name than video thumnail
  fs.unlink(CONFIG.STORAGE.PREVIEWS_DIR + video.getPreviewName(), callback)
}

function createTorrentFromVideo (video, videoPath, callback) {
  const options = {
    announceList: [
      [ CONFIG.WEBSERVER.WS + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT + '/tracker/socket' ]
    ],
    urlList: [
      CONFIG.WEBSERVER.URL + STATIC_PATHS.WEBSEED + video.getVideoFilename()
    ]
  }

  createTorrent(videoPath, options, function (err, torrent) {
    if (err) return callback(err)

    const filePath = join(CONFIG.STORAGE.TORRENTS_DIR, video.getTorrentName())
    fs.writeFile(filePath, torrent, function (err) {
      if (err) return callback(err)

      const parsedTorrent = parseTorrent(torrent)
      video.set('infoHash', parsedTorrent.infoHash)
      video.validate().asCallback(callback)
    })
  })
}

function createPreview (video, videoPath, callback) {
  generateImage(video, videoPath, CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName(), callback)
}

function createThumbnail (video, videoPath, callback) {
  generateImage(video, videoPath, CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName(), THUMBNAILS_SIZE, callback)
}

function generateImage (video, videoPath, folder, imageName, size, callback?) {
  const options: any = {
    filename: imageName,
    count: 1,
    folder
  }

  if (!callback) {
    callback = size
  } else {
    options.size = size
  }

  ffmpeg(videoPath)
    .on('error', callback)
    .on('end', function () {
      callback(null, imageName)
    })
    .thumbnail(options)
}

function removeFromBlacklist (video, callback) {
  // Find the blacklisted video
  db.BlacklistedVideo.loadByVideoId(video.id, function (err, video) {
    // If an error occured, stop here
    if (err) {
      logger.error('Error when fetching video from blacklist.', { error: err })
      return callback(err)
    }

    // If we found the video, remove it from the blacklist
    if (video) {
      video.destroy().asCallback(callback)
    } else {
      // If haven't found it, simply ignore it and do nothing
      return callback()
    }
  })
}