mirror of
https://github.com/Chocobozzz/PeerTube.git
synced 2024-08-17 01:10:34 +03:00
Correctly handle large HLS files for redundancy
This commit is contained in:
parent
a1c63fe1a2
commit
18998c45c0
@ -1,4 +1,4 @@
|
||||
import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra'
|
||||
import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, stat, writeFile } from 'fs-extra'
|
||||
import { flatten, uniq } from 'lodash'
|
||||
import { basename, dirname, join } from 'path'
|
||||
import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models'
|
||||
@ -108,8 +108,9 @@ async function buildSha256Segment (segmentPath: string) {
|
||||
return sha256(buf)
|
||||
}
|
||||
|
||||
function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) {
|
||||
function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number, bodyKBLimit: number) {
|
||||
let timer
|
||||
let remainingBodyKBLimit = bodyKBLimit
|
||||
|
||||
logger.info('Importing HLS playlist %s', playlistUrl)
|
||||
|
||||
@ -136,8 +137,12 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string,
|
||||
for (const fileUrl of fileUrls) {
|
||||
const destPath = join(tmpDirectory, basename(fileUrl))
|
||||
|
||||
const bodyKBLimit = 10 * 1000 * 1000 // 10GB
|
||||
await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit })
|
||||
await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit })
|
||||
|
||||
const { size } = await stat(destPath)
|
||||
remainingBodyKBLimit -= (size / 1000)
|
||||
|
||||
logger.debug('Downloaded HLS playlist file %s with %d kB remained limit.', fileUrl, Math.floor(remainingBodyKBLimit))
|
||||
}
|
||||
|
||||
clearTimeout(timer)
|
||||
|
@ -4,9 +4,7 @@ import { getServerActor } from '@server/models/application/application'
|
||||
import { TrackerModel } from '@server/models/server/tracker'
|
||||
import { VideoModel } from '@server/models/video/video'
|
||||
import {
|
||||
MStreamingPlaylist,
|
||||
MStreamingPlaylistFiles,
|
||||
MStreamingPlaylistVideo,
|
||||
MVideoAccountLight,
|
||||
MVideoFile,
|
||||
MVideoFileVideo,
|
||||
@ -249,7 +247,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
|
||||
private async createStreamingPlaylistRedundancy (
|
||||
redundancy: VideosRedundancyStrategy,
|
||||
video: MVideoAccountLight,
|
||||
playlistArg: MStreamingPlaylist
|
||||
playlistArg: MStreamingPlaylistFiles
|
||||
) {
|
||||
let strategy = 'manual'
|
||||
let expiresOn: Date = null
|
||||
@ -259,16 +257,17 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
|
||||
expiresOn = this.buildNewExpiration(redundancy.minLifetime)
|
||||
}
|
||||
|
||||
const playlist = playlistArg as MStreamingPlaylistVideo
|
||||
playlist.Video = video
|
||||
|
||||
const playlist = Object.assign(playlistArg, { Video: video })
|
||||
const serverActor = await getServerActor()
|
||||
|
||||
logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy)
|
||||
|
||||
const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
|
||||
const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video)
|
||||
await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT)
|
||||
|
||||
const maxSizeKB = this.getTotalFileSizes([], [ playlist ]) / 1000
|
||||
const toleranceKB = maxSizeKB + ((5 * maxSizeKB) / 100) // 5% more tolerance
|
||||
await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT, toleranceKB)
|
||||
|
||||
const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({
|
||||
expiresOn,
|
||||
@ -334,7 +333,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
|
||||
return `${object.VideoStreamingPlaylist.getMasterPlaylistUrl(object.VideoStreamingPlaylist.Video)}`
|
||||
}
|
||||
|
||||
private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]) {
|
||||
private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]): number {
|
||||
const fileReducer = (previous: number, current: MVideoFile) => previous + current.size
|
||||
|
||||
let allFiles = files
|
||||
|
Loading…
Reference in New Issue
Block a user