mirror of
https://github.com/Chocobozzz/PeerTube.git
synced 2024-09-11 13:37:10 +03:00
Implement remote runner jobs in server
Move ffmpeg functions to @shared
This commit is contained in:
parent
6bcb854cde
commit
0c9668f779
@ -375,6 +375,12 @@ feeds:
|
||||
# Default number of comments displayed in feeds
|
||||
count: 20
|
||||
|
||||
remote_runners:
|
||||
# Consider jobs that are processed by a remote runner as stalled after this period of time without any update
|
||||
stalled_jobs:
|
||||
live: '30 seconds'
|
||||
vod: '2 minutes'
|
||||
|
||||
cache:
|
||||
previews:
|
||||
size: 500 # Max number of previews you want to cache
|
||||
@ -433,12 +439,18 @@ transcoding:
|
||||
# If a user uploads an audio file, PeerTube will create a video by merging the preview file and the audio file
|
||||
allow_audio_files: true
|
||||
|
||||
# Amount of threads used by ffmpeg for 1 transcoding job
|
||||
# Enable remote runners to transcode your videos
|
||||
# If enabled, your instance won't transcode the videos itself
|
||||
# At least 1 remote runner must be configured to transcode your videos
|
||||
remote_runners:
|
||||
enabled: false
|
||||
|
||||
# Amount of threads used by ffmpeg for 1 local transcoding job
|
||||
threads: 1
|
||||
# Amount of transcoding jobs to execute in parallel
|
||||
# Amount of local transcoding jobs to execute in parallel
|
||||
concurrency: 1
|
||||
|
||||
# Choose the transcoding profile
|
||||
# Choose the local transcoding profile
|
||||
# New profiles can be added by plugins
|
||||
# Available in core PeerTube: 'default'
|
||||
profile: 'default'
|
||||
@ -533,9 +545,17 @@ live:
|
||||
# Allow to transcode the live streaming in multiple live resolutions
|
||||
transcoding:
|
||||
enabled: true
|
||||
|
||||
# Enable remote runners to transcode your videos
|
||||
# If enabled, your instance won't transcode the videos itself
|
||||
# At least 1 remote runner must be configured to transcode your videos
|
||||
remote_runners:
|
||||
enabled: false
|
||||
|
||||
# Amount of threads used by ffmpeg per live when using local transcoding
|
||||
threads: 2
|
||||
|
||||
# Choose the transcoding profile
|
||||
# Choose the local transcoding profile
|
||||
# New profiles can be added by plugins
|
||||
# Available in core PeerTube: 'default'
|
||||
profile: 'default'
|
||||
@ -754,7 +774,7 @@ search:
|
||||
search_index:
|
||||
enabled: false
|
||||
# URL of the search index, that should use the same search API and routes
|
||||
# than PeerTube: https://docs.joinpeertube.org/api/rest-reference.html
|
||||
# than PeerTube: https://docs.joinpeertube.org/api-rest-reference.html
|
||||
# You should deploy your own with https://framagit.org/framasoft/peertube/search-index,
|
||||
# and can use https://search.joinpeertube.org/ for tests, but keep in mind the latter is an unmoderated search index
|
||||
url: ''
|
||||
|
@ -373,6 +373,12 @@ feeds:
|
||||
# Default number of comments displayed in feeds
|
||||
count: 20
|
||||
|
||||
remote_runners:
|
||||
# Consider jobs that are processed by a remote runner as stalled after this period of time without any update
|
||||
stalled_jobs:
|
||||
live: '30 seconds'
|
||||
vod: '2 minutes'
|
||||
|
||||
###############################################################################
|
||||
#
|
||||
# From this point, almost all following keys can be overridden by the web interface
|
||||
@ -443,12 +449,18 @@ transcoding:
|
||||
# If a user uploads an audio file, PeerTube will create a video by merging the preview file and the audio file
|
||||
allow_audio_files: true
|
||||
|
||||
# Amount of threads used by ffmpeg for 1 transcoding job
|
||||
# Enable remote runners to transcode your videos
|
||||
# If enabled, your instance won't transcode the videos itself
|
||||
# At least 1 remote runner must be configured to transcode your videos
|
||||
remote_runners:
|
||||
enabled: false
|
||||
|
||||
# Amount of threads used by ffmpeg for 1 local transcoding job
|
||||
threads: 1
|
||||
# Amount of transcoding jobs to execute in parallel
|
||||
# Amount of local transcoding jobs to execute in parallel
|
||||
concurrency: 1
|
||||
|
||||
# Choose the transcoding profile
|
||||
# Choose the local transcoding profile
|
||||
# New profiles can be added by plugins
|
||||
# Available in core PeerTube: 'default'
|
||||
profile: 'default'
|
||||
@ -543,9 +555,17 @@ live:
|
||||
# Allow to transcode the live streaming in multiple live resolutions
|
||||
transcoding:
|
||||
enabled: true
|
||||
|
||||
# Enable remote runners to transcode your videos
|
||||
# If enabled, your instance won't transcode the videos itself
|
||||
# At least 1 remote runner must be configured to transcode your videos
|
||||
remote_runners:
|
||||
enabled: false
|
||||
|
||||
# Amount of threads used by ffmpeg per live when using local transcoding
|
||||
threads: 2
|
||||
|
||||
# Choose the transcoding profile
|
||||
# Choose the local transcoding profile
|
||||
# New profiles can be added by plugins
|
||||
# Available in core PeerTube: 'default'
|
||||
profile: 'default'
|
||||
@ -607,7 +627,7 @@ import:
|
||||
# See https://docs.joinpeertube.org/maintain/configuration#security for more information
|
||||
enabled: false
|
||||
|
||||
# Add ability for your users to synchronize their channels with external channels, playlists, etc.
|
||||
# Add ability for your users to synchronize their channels with external channels, playlists, etc
|
||||
video_channel_synchronization:
|
||||
enabled: false
|
||||
|
||||
@ -768,9 +788,9 @@ search:
|
||||
# You should deploy your own with https://framagit.org/framasoft/peertube/search-index,
|
||||
# and can use https://search.joinpeertube.org/ for tests, but keep in mind the latter is an unmoderated search index
|
||||
url: ''
|
||||
# You can disable local search, so users only use the search index
|
||||
# You can disable local search in the client, so users only use the search index
|
||||
disable_local_search: false
|
||||
# If you did not disable local search, you can decide to use the search index by default
|
||||
# If you did not disable local search in the client, you can decide to use the search index by default
|
||||
is_default_search: false
|
||||
|
||||
# PeerTube client/interface configuration
|
||||
|
@ -133,6 +133,7 @@ import { AutoFollowIndexInstances } from './server/lib/schedulers/auto-follow-in
|
||||
import { RemoveDanglingResumableUploadsScheduler } from './server/lib/schedulers/remove-dangling-resumable-uploads-scheduler'
|
||||
import { VideoViewsBufferScheduler } from './server/lib/schedulers/video-views-buffer-scheduler'
|
||||
import { GeoIPUpdateScheduler } from './server/lib/schedulers/geo-ip-update-scheduler'
|
||||
import { RunnerJobWatchDogScheduler } from './server/lib/schedulers/runner-job-watch-dog-scheduler'
|
||||
import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto'
|
||||
import { PeerTubeSocket } from './server/lib/peertube-socket'
|
||||
import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls'
|
||||
@ -331,6 +332,7 @@ async function startApplication () {
|
||||
VideoChannelSyncLatestScheduler.Instance.enable()
|
||||
VideoViewsBufferScheduler.Instance.enable()
|
||||
GeoIPUpdateScheduler.Instance.enable()
|
||||
RunnerJobWatchDogScheduler.Instance.enable()
|
||||
|
||||
OpenTelemetryMetrics.Instance.registerMetrics({ trackerServer })
|
||||
|
||||
|
@ -217,6 +217,9 @@ function customConfig (): CustomConfig {
|
||||
},
|
||||
transcoding: {
|
||||
enabled: CONFIG.TRANSCODING.ENABLED,
|
||||
remoteRunners: {
|
||||
enabled: CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED
|
||||
},
|
||||
allowAdditionalExtensions: CONFIG.TRANSCODING.ALLOW_ADDITIONAL_EXTENSIONS,
|
||||
allowAudioFiles: CONFIG.TRANSCODING.ALLOW_AUDIO_FILES,
|
||||
threads: CONFIG.TRANSCODING.THREADS,
|
||||
@ -252,6 +255,9 @@ function customConfig (): CustomConfig {
|
||||
maxUserLives: CONFIG.LIVE.MAX_USER_LIVES,
|
||||
transcoding: {
|
||||
enabled: CONFIG.LIVE.TRANSCODING.ENABLED,
|
||||
remoteRunners: {
|
||||
enabled: CONFIG.LIVE.TRANSCODING.REMOTE_RUNNERS.ENABLED
|
||||
},
|
||||
threads: CONFIG.LIVE.TRANSCODING.THREADS,
|
||||
profile: CONFIG.LIVE.TRANSCODING.PROFILE,
|
||||
resolutions: {
|
||||
|
@ -15,6 +15,7 @@ import { metricsRouter } from './metrics'
|
||||
import { oauthClientsRouter } from './oauth-clients'
|
||||
import { overviewsRouter } from './overviews'
|
||||
import { pluginRouter } from './plugins'
|
||||
import { runnersRouter } from './runners'
|
||||
import { searchRouter } from './search'
|
||||
import { serverRouter } from './server'
|
||||
import { usersRouter } from './users'
|
||||
@ -55,6 +56,7 @@ apiRouter.use('/overviews', overviewsRouter)
|
||||
apiRouter.use('/plugins', pluginRouter)
|
||||
apiRouter.use('/custom-pages', customPageRouter)
|
||||
apiRouter.use('/blocklist', blocklistRouter)
|
||||
apiRouter.use('/runners', runnersRouter)
|
||||
apiRouter.use('/ping', pong)
|
||||
apiRouter.use('/*', badRequest)
|
||||
|
||||
|
@ -93,6 +93,9 @@ async function formatJob (job: BullJob, state?: JobState): Promise<Job> {
|
||||
state: state || await job.getState(),
|
||||
type: job.queueName as JobType,
|
||||
data: job.data,
|
||||
parent: job.parent
|
||||
? { id: job.parent.id }
|
||||
: undefined,
|
||||
progress: job.progress as number,
|
||||
priority: job.opts.priority,
|
||||
error,
|
||||
|
18
server/controllers/api/runners/index.ts
Normal file
18
server/controllers/api/runners/index.ts
Normal file
@ -0,0 +1,18 @@
|
||||
import express from 'express'
|
||||
import { runnerJobsRouter } from './jobs'
|
||||
import { runnerJobFilesRouter } from './jobs-files'
|
||||
import { manageRunnersRouter } from './manage-runners'
|
||||
import { runnerRegistrationTokensRouter } from './registration-tokens'
|
||||
|
||||
const runnersRouter = express.Router()
|
||||
|
||||
runnersRouter.use('/', manageRunnersRouter)
|
||||
runnersRouter.use('/', runnerJobsRouter)
|
||||
runnersRouter.use('/', runnerJobFilesRouter)
|
||||
runnersRouter.use('/', runnerRegistrationTokensRouter)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
runnersRouter
|
||||
}
|
84
server/controllers/api/runners/jobs-files.ts
Normal file
84
server/controllers/api/runners/jobs-files.ts
Normal file
@ -0,0 +1,84 @@
|
||||
import express from 'express'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger'
|
||||
import { proxifyHLS, proxifyWebTorrentFile } from '@server/lib/object-storage'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager'
|
||||
import { asyncMiddleware } from '@server/middlewares'
|
||||
import { jobOfRunnerGetValidator } from '@server/middlewares/validators/runners'
|
||||
import { runnerJobGetVideoTranscodingFileValidator } from '@server/middlewares/validators/runners/job-files'
|
||||
import { VideoStorage } from '@shared/models'
|
||||
|
||||
const lTags = loggerTagsFactory('api', 'runner')
|
||||
|
||||
const runnerJobFilesRouter = express.Router()
|
||||
|
||||
runnerJobFilesRouter.post('/jobs/:jobUUID/files/videos/:videoId/max-quality',
|
||||
asyncMiddleware(jobOfRunnerGetValidator),
|
||||
asyncMiddleware(runnerJobGetVideoTranscodingFileValidator),
|
||||
asyncMiddleware(getMaxQualityVideoFile)
|
||||
)
|
||||
|
||||
runnerJobFilesRouter.post('/jobs/:jobUUID/files/videos/:videoId/previews/max-quality',
|
||||
asyncMiddleware(jobOfRunnerGetValidator),
|
||||
asyncMiddleware(runnerJobGetVideoTranscodingFileValidator),
|
||||
getMaxQualityVideoPreview
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
runnerJobFilesRouter
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function getMaxQualityVideoFile (req: express.Request, res: express.Response) {
|
||||
const runnerJob = res.locals.runnerJob
|
||||
const runner = runnerJob.Runner
|
||||
const video = res.locals.videoAll
|
||||
|
||||
logger.info(
|
||||
'Get max quality file of video %s of job %s for runner %s', video.uuid, runnerJob.uuid, runner.name,
|
||||
lTags(runner.name, runnerJob.id, runnerJob.type)
|
||||
)
|
||||
|
||||
const file = video.getMaxQualityFile()
|
||||
|
||||
if (file.storage === VideoStorage.OBJECT_STORAGE) {
|
||||
if (file.isHLS()) {
|
||||
return proxifyHLS({
|
||||
req,
|
||||
res,
|
||||
filename: file.filename,
|
||||
playlist: video.getHLSPlaylist(),
|
||||
reinjectVideoFileToken: false,
|
||||
video
|
||||
})
|
||||
}
|
||||
|
||||
// Web video
|
||||
return proxifyWebTorrentFile({
|
||||
req,
|
||||
res,
|
||||
filename: file.filename
|
||||
})
|
||||
}
|
||||
|
||||
return VideoPathManager.Instance.makeAvailableVideoFile(file, videoPath => {
|
||||
return res.sendFile(videoPath)
|
||||
})
|
||||
}
|
||||
|
||||
function getMaxQualityVideoPreview (req: express.Request, res: express.Response) {
|
||||
const runnerJob = res.locals.runnerJob
|
||||
const runner = runnerJob.Runner
|
||||
const video = res.locals.videoAll
|
||||
|
||||
logger.info(
|
||||
'Get max quality preview file of video %s of job %s for runner %s', video.uuid, runnerJob.uuid, runner.name,
|
||||
lTags(runner.name, runnerJob.id, runnerJob.type)
|
||||
)
|
||||
|
||||
const file = video.getPreview()
|
||||
|
||||
return res.sendFile(file.getPath())
|
||||
}
|
352
server/controllers/api/runners/jobs.ts
Normal file
352
server/controllers/api/runners/jobs.ts
Normal file
@ -0,0 +1,352 @@
|
||||
import express, { UploadFiles } from 'express'
|
||||
import { createReqFiles } from '@server/helpers/express-utils'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger'
|
||||
import { generateRunnerJobToken } from '@server/helpers/token-generator'
|
||||
import { MIMETYPES } from '@server/initializers/constants'
|
||||
import { sequelizeTypescript } from '@server/initializers/database'
|
||||
import { getRunnerJobHandlerClass, updateLastRunnerContact } from '@server/lib/runners'
|
||||
import {
|
||||
asyncMiddleware,
|
||||
authenticate,
|
||||
ensureUserHasRight,
|
||||
paginationValidator,
|
||||
runnerJobsSortValidator,
|
||||
setDefaultPagination,
|
||||
setDefaultSort
|
||||
} from '@server/middlewares'
|
||||
import {
|
||||
abortRunnerJobValidator,
|
||||
acceptRunnerJobValidator,
|
||||
errorRunnerJobValidator,
|
||||
getRunnerFromTokenValidator,
|
||||
jobOfRunnerGetValidator,
|
||||
runnerJobGetValidator,
|
||||
successRunnerJobValidator,
|
||||
updateRunnerJobValidator
|
||||
} from '@server/middlewares/validators/runners'
|
||||
import { RunnerModel } from '@server/models/runner/runner'
|
||||
import { RunnerJobModel } from '@server/models/runner/runner-job'
|
||||
import {
|
||||
AbortRunnerJobBody,
|
||||
AcceptRunnerJobResult,
|
||||
ErrorRunnerJobBody,
|
||||
HttpStatusCode,
|
||||
ListRunnerJobsQuery,
|
||||
LiveRTMPHLSTranscodingUpdatePayload,
|
||||
RequestRunnerJobResult,
|
||||
RunnerJobState,
|
||||
RunnerJobSuccessBody,
|
||||
RunnerJobSuccessPayload,
|
||||
RunnerJobType,
|
||||
RunnerJobUpdateBody,
|
||||
RunnerJobUpdatePayload,
|
||||
UserRight,
|
||||
VODAudioMergeTranscodingSuccess,
|
||||
VODHLSTranscodingSuccess,
|
||||
VODWebVideoTranscodingSuccess
|
||||
} from '@shared/models'
|
||||
|
||||
const postRunnerJobSuccessVideoFiles = createReqFiles(
|
||||
[ 'payload[videoFile]', 'payload[resolutionPlaylistFile]' ],
|
||||
{ ...MIMETYPES.VIDEO.MIMETYPE_EXT, ...MIMETYPES.M3U8.MIMETYPE_EXT }
|
||||
)
|
||||
|
||||
const runnerJobUpdateVideoFiles = createReqFiles(
|
||||
[ 'payload[videoChunkFile]', 'payload[resolutionPlaylistFile]', 'payload[masterPlaylistFile]' ],
|
||||
{ ...MIMETYPES.VIDEO.MIMETYPE_EXT, ...MIMETYPES.M3U8.MIMETYPE_EXT }
|
||||
)
|
||||
|
||||
const lTags = loggerTagsFactory('api', 'runner')
|
||||
|
||||
const runnerJobsRouter = express.Router()
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Controllers for runners
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
runnerJobsRouter.post('/jobs/request',
|
||||
asyncMiddleware(getRunnerFromTokenValidator),
|
||||
asyncMiddleware(requestRunnerJob)
|
||||
)
|
||||
|
||||
runnerJobsRouter.post('/jobs/:jobUUID/accept',
|
||||
asyncMiddleware(runnerJobGetValidator),
|
||||
acceptRunnerJobValidator,
|
||||
asyncMiddleware(getRunnerFromTokenValidator),
|
||||
asyncMiddleware(acceptRunnerJob)
|
||||
)
|
||||
|
||||
runnerJobsRouter.post('/jobs/:jobUUID/abort',
|
||||
asyncMiddleware(jobOfRunnerGetValidator),
|
||||
abortRunnerJobValidator,
|
||||
asyncMiddleware(abortRunnerJob)
|
||||
)
|
||||
|
||||
runnerJobsRouter.post('/jobs/:jobUUID/update',
|
||||
runnerJobUpdateVideoFiles,
|
||||
asyncMiddleware(jobOfRunnerGetValidator),
|
||||
updateRunnerJobValidator,
|
||||
asyncMiddleware(updateRunnerJobController)
|
||||
)
|
||||
|
||||
runnerJobsRouter.post('/jobs/:jobUUID/error',
|
||||
asyncMiddleware(jobOfRunnerGetValidator),
|
||||
errorRunnerJobValidator,
|
||||
asyncMiddleware(errorRunnerJob)
|
||||
)
|
||||
|
||||
runnerJobsRouter.post('/jobs/:jobUUID/success',
|
||||
postRunnerJobSuccessVideoFiles,
|
||||
asyncMiddleware(jobOfRunnerGetValidator),
|
||||
successRunnerJobValidator,
|
||||
asyncMiddleware(postRunnerJobSuccess)
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Controllers for admins
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
runnerJobsRouter.post('/jobs/:jobUUID/cancel',
|
||||
authenticate,
|
||||
ensureUserHasRight(UserRight.MANAGE_RUNNERS),
|
||||
asyncMiddleware(runnerJobGetValidator),
|
||||
asyncMiddleware(cancelRunnerJob)
|
||||
)
|
||||
|
||||
runnerJobsRouter.get('/jobs',
|
||||
authenticate,
|
||||
ensureUserHasRight(UserRight.MANAGE_RUNNERS),
|
||||
paginationValidator,
|
||||
runnerJobsSortValidator,
|
||||
setDefaultSort,
|
||||
setDefaultPagination,
|
||||
asyncMiddleware(listRunnerJobs)
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
runnerJobsRouter
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Controllers for runners
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function requestRunnerJob (req: express.Request, res: express.Response) {
|
||||
const runner = res.locals.runner
|
||||
const availableJobs = await RunnerJobModel.listAvailableJobs()
|
||||
|
||||
logger.debug('Runner %s requests for a job.', runner.name, { availableJobs, ...lTags(runner.name) })
|
||||
|
||||
const result: RequestRunnerJobResult = {
|
||||
availableJobs: availableJobs.map(j => ({
|
||||
uuid: j.uuid,
|
||||
type: j.type,
|
||||
payload: j.payload
|
||||
}))
|
||||
}
|
||||
|
||||
updateLastRunnerContact(req, runner)
|
||||
|
||||
return res.json(result)
|
||||
}
|
||||
|
||||
async function acceptRunnerJob (req: express.Request, res: express.Response) {
|
||||
const runner = res.locals.runner
|
||||
const runnerJob = res.locals.runnerJob
|
||||
|
||||
runnerJob.state = RunnerJobState.PROCESSING
|
||||
runnerJob.processingJobToken = generateRunnerJobToken()
|
||||
runnerJob.startedAt = new Date()
|
||||
runnerJob.runnerId = runner.id
|
||||
|
||||
const newRunnerJob = await sequelizeTypescript.transaction(transaction => {
|
||||
return runnerJob.save({ transaction })
|
||||
})
|
||||
newRunnerJob.Runner = runner as RunnerModel
|
||||
|
||||
const result: AcceptRunnerJobResult = {
|
||||
job: {
|
||||
...newRunnerJob.toFormattedJSON(),
|
||||
|
||||
jobToken: newRunnerJob.processingJobToken
|
||||
}
|
||||
}
|
||||
|
||||
updateLastRunnerContact(req, runner)
|
||||
|
||||
logger.info(
|
||||
'Remote runner %s has accepted job %s (%s)', runner.name, runnerJob.uuid, runnerJob.type,
|
||||
lTags(runner.name, runnerJob.uuid, runnerJob.type)
|
||||
)
|
||||
|
||||
return res.json(result)
|
||||
}
|
||||
|
||||
async function abortRunnerJob (req: express.Request, res: express.Response) {
|
||||
const runnerJob = res.locals.runnerJob
|
||||
const runner = runnerJob.Runner
|
||||
const body: AbortRunnerJobBody = req.body
|
||||
|
||||
logger.info(
|
||||
'Remote runner %s is aborting job %s (%s)', runner.name, runnerJob.uuid, runnerJob.type,
|
||||
{ reason: body.reason, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) }
|
||||
)
|
||||
|
||||
const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob)
|
||||
await new RunnerJobHandler().abort({ runnerJob })
|
||||
|
||||
updateLastRunnerContact(req, runnerJob.Runner)
|
||||
|
||||
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
|
||||
}
|
||||
|
||||
async function errorRunnerJob (req: express.Request, res: express.Response) {
|
||||
const runnerJob = res.locals.runnerJob
|
||||
const runner = runnerJob.Runner
|
||||
const body: ErrorRunnerJobBody = req.body
|
||||
|
||||
runnerJob.failures += 1
|
||||
|
||||
logger.error(
|
||||
'Remote runner %s had an error with job %s (%s)', runner.name, runnerJob.uuid, runnerJob.type,
|
||||
{ errorMessage: body.message, totalFailures: runnerJob.failures, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) }
|
||||
)
|
||||
|
||||
const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob)
|
||||
await new RunnerJobHandler().error({ runnerJob, message: body.message })
|
||||
|
||||
updateLastRunnerContact(req, runnerJob.Runner)
|
||||
|
||||
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const jobUpdateBuilders: {
|
||||
[id in RunnerJobType]?: (payload: RunnerJobUpdatePayload, files?: UploadFiles) => RunnerJobUpdatePayload
|
||||
} = {
|
||||
'live-rtmp-hls-transcoding': (payload: LiveRTMPHLSTranscodingUpdatePayload, files) => {
|
||||
return {
|
||||
...payload,
|
||||
|
||||
masterPlaylistFile: files['payload[masterPlaylistFile]']?.[0].path,
|
||||
resolutionPlaylistFile: files['payload[resolutionPlaylistFile]']?.[0].path,
|
||||
videoChunkFile: files['payload[videoChunkFile]']?.[0].path
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function updateRunnerJobController (req: express.Request, res: express.Response) {
|
||||
const runnerJob = res.locals.runnerJob
|
||||
const runner = runnerJob.Runner
|
||||
const body: RunnerJobUpdateBody = req.body
|
||||
|
||||
const payloadBuilder = jobUpdateBuilders[runnerJob.type]
|
||||
const updatePayload = payloadBuilder
|
||||
? payloadBuilder(body.payload, req.files as UploadFiles)
|
||||
: undefined
|
||||
|
||||
logger.debug(
|
||||
'Remote runner %s is updating job %s (%s)', runnerJob.Runner.name, runnerJob.uuid, runnerJob.type,
|
||||
{ body, updatePayload, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) }
|
||||
)
|
||||
|
||||
const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob)
|
||||
await new RunnerJobHandler().update({
|
||||
runnerJob,
|
||||
progress: req.body.progress,
|
||||
updatePayload
|
||||
})
|
||||
|
||||
updateLastRunnerContact(req, runnerJob.Runner)
|
||||
|
||||
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const jobSuccessPayloadBuilders: {
|
||||
[id in RunnerJobType]: (payload: RunnerJobSuccessPayload, files?: UploadFiles) => RunnerJobSuccessPayload
|
||||
} = {
|
||||
'vod-web-video-transcoding': (payload: VODWebVideoTranscodingSuccess, files) => {
|
||||
return {
|
||||
...payload,
|
||||
|
||||
videoFile: files['payload[videoFile]'][0].path
|
||||
}
|
||||
},
|
||||
|
||||
'vod-hls-transcoding': (payload: VODHLSTranscodingSuccess, files) => {
|
||||
return {
|
||||
...payload,
|
||||
|
||||
videoFile: files['payload[videoFile]'][0].path,
|
||||
resolutionPlaylistFile: files['payload[resolutionPlaylistFile]'][0].path
|
||||
}
|
||||
},
|
||||
|
||||
'vod-audio-merge-transcoding': (payload: VODAudioMergeTranscodingSuccess, files) => {
|
||||
return {
|
||||
...payload,
|
||||
|
||||
videoFile: files['payload[videoFile]'][0].path
|
||||
}
|
||||
},
|
||||
|
||||
'live-rtmp-hls-transcoding': () => ({})
|
||||
}
|
||||
|
||||
async function postRunnerJobSuccess (req: express.Request, res: express.Response) {
|
||||
const runnerJob = res.locals.runnerJob
|
||||
const runner = runnerJob.Runner
|
||||
const body: RunnerJobSuccessBody = req.body
|
||||
|
||||
const resultPayload = jobSuccessPayloadBuilders[runnerJob.type](body.payload, req.files as UploadFiles)
|
||||
|
||||
logger.info(
|
||||
'Remote runner %s is sending success result for job %s (%s)', runnerJob.Runner.name, runnerJob.uuid, runnerJob.type,
|
||||
{ resultPayload, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) }
|
||||
)
|
||||
|
||||
const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob)
|
||||
await new RunnerJobHandler().complete({ runnerJob, resultPayload })
|
||||
|
||||
updateLastRunnerContact(req, runnerJob.Runner)
|
||||
|
||||
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Controllers for admins
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function cancelRunnerJob (req: express.Request, res: express.Response) {
|
||||
const runnerJob = res.locals.runnerJob
|
||||
|
||||
logger.info('Cancelling job %s (%s)', runnerJob.type, lTags(runnerJob.uuid, runnerJob.type))
|
||||
|
||||
const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob)
|
||||
await new RunnerJobHandler().cancel({ runnerJob })
|
||||
|
||||
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
|
||||
}
|
||||
|
||||
async function listRunnerJobs (req: express.Request, res: express.Response) {
|
||||
const query: ListRunnerJobsQuery = req.query
|
||||
|
||||
const resultList = await RunnerJobModel.listForApi({
|
||||
start: query.start,
|
||||
count: query.count,
|
||||
sort: query.sort,
|
||||
search: query.search
|
||||
})
|
||||
|
||||
return res.json({
|
||||
total: resultList.total,
|
||||
data: resultList.data.map(d => d.toFormattedAdminJSON())
|
||||
})
|
||||
}
|
107
server/controllers/api/runners/manage-runners.ts
Normal file
107
server/controllers/api/runners/manage-runners.ts
Normal file
@ -0,0 +1,107 @@
|
||||
import express from 'express'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger'
|
||||
import { generateRunnerToken } from '@server/helpers/token-generator'
|
||||
import {
|
||||
asyncMiddleware,
|
||||
authenticate,
|
||||
ensureUserHasRight,
|
||||
paginationValidator,
|
||||
runnersSortValidator,
|
||||
setDefaultPagination,
|
||||
setDefaultSort
|
||||
} from '@server/middlewares'
|
||||
import { deleteRunnerValidator, getRunnerFromTokenValidator, registerRunnerValidator } from '@server/middlewares/validators/runners'
|
||||
import { RunnerModel } from '@server/models/runner/runner'
|
||||
import { HttpStatusCode, ListRunnersQuery, RegisterRunnerBody, UserRight } from '@shared/models'
|
||||
|
||||
const lTags = loggerTagsFactory('api', 'runner')
|
||||
|
||||
const manageRunnersRouter = express.Router()
|
||||
|
||||
manageRunnersRouter.post('/register',
|
||||
asyncMiddleware(registerRunnerValidator),
|
||||
asyncMiddleware(registerRunner)
|
||||
)
|
||||
manageRunnersRouter.post('/unregister',
|
||||
asyncMiddleware(getRunnerFromTokenValidator),
|
||||
asyncMiddleware(unregisterRunner)
|
||||
)
|
||||
|
||||
manageRunnersRouter.delete('/:runnerId',
|
||||
authenticate,
|
||||
ensureUserHasRight(UserRight.MANAGE_RUNNERS),
|
||||
asyncMiddleware(deleteRunnerValidator),
|
||||
asyncMiddleware(deleteRunner)
|
||||
)
|
||||
|
||||
manageRunnersRouter.get('/',
|
||||
authenticate,
|
||||
ensureUserHasRight(UserRight.MANAGE_RUNNERS),
|
||||
paginationValidator,
|
||||
runnersSortValidator,
|
||||
setDefaultSort,
|
||||
setDefaultPagination,
|
||||
asyncMiddleware(listRunners)
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
manageRunnersRouter
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function registerRunner (req: express.Request, res: express.Response) {
|
||||
const body: RegisterRunnerBody = req.body
|
||||
|
||||
const runnerToken = generateRunnerToken()
|
||||
|
||||
const runner = new RunnerModel({
|
||||
runnerToken,
|
||||
name: body.name,
|
||||
description: body.description,
|
||||
lastContact: new Date(),
|
||||
ip: req.ip,
|
||||
runnerRegistrationTokenId: res.locals.runnerRegistrationToken.id
|
||||
})
|
||||
|
||||
await runner.save()
|
||||
|
||||
logger.info('Registered new runner %s', runner.name, { ...lTags(runner.name) })
|
||||
|
||||
return res.json({ id: runner.id, runnerToken })
|
||||
}
|
||||
async function unregisterRunner (req: express.Request, res: express.Response) {
|
||||
const runner = res.locals.runner
|
||||
await runner.destroy()
|
||||
|
||||
logger.info('Unregistered runner %s', runner.name, { ...lTags(runner.name) })
|
||||
|
||||
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
|
||||
}
|
||||
|
||||
async function deleteRunner (req: express.Request, res: express.Response) {
|
||||
const runner = res.locals.runner
|
||||
|
||||
await runner.destroy()
|
||||
|
||||
logger.info('Deleted runner %s', runner.name, { ...lTags(runner.name) })
|
||||
|
||||
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
|
||||
}
|
||||
|
||||
async function listRunners (req: express.Request, res: express.Response) {
|
||||
const query: ListRunnersQuery = req.query
|
||||
|
||||
const resultList = await RunnerModel.listForApi({
|
||||
start: query.start,
|
||||
count: query.count,
|
||||
sort: query.sort
|
||||
})
|
||||
|
||||
return res.json({
|
||||
total: resultList.total,
|
||||
data: resultList.data.map(d => d.toFormattedJSON())
|
||||
})
|
||||
}
|
87
server/controllers/api/runners/registration-tokens.ts
Normal file
87
server/controllers/api/runners/registration-tokens.ts
Normal file
@ -0,0 +1,87 @@
|
||||
import express from 'express'
|
||||
import { generateRunnerRegistrationToken } from '@server/helpers/token-generator'
|
||||
import {
|
||||
asyncMiddleware,
|
||||
authenticate,
|
||||
ensureUserHasRight,
|
||||
paginationValidator,
|
||||
runnerRegistrationTokensSortValidator,
|
||||
setDefaultPagination,
|
||||
setDefaultSort
|
||||
} from '@server/middlewares'
|
||||
import { deleteRegistrationTokenValidator } from '@server/middlewares/validators/runners'
|
||||
import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token'
|
||||
import { HttpStatusCode, ListRunnerRegistrationTokensQuery, UserRight } from '@shared/models'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger'
|
||||
|
||||
const lTags = loggerTagsFactory('api', 'runner')
|
||||
|
||||
const runnerRegistrationTokensRouter = express.Router()
|
||||
|
||||
runnerRegistrationTokensRouter.post('/registration-tokens/generate',
|
||||
authenticate,
|
||||
ensureUserHasRight(UserRight.MANAGE_RUNNERS),
|
||||
asyncMiddleware(generateRegistrationToken)
|
||||
)
|
||||
|
||||
runnerRegistrationTokensRouter.delete('/registration-tokens/:id',
|
||||
authenticate,
|
||||
ensureUserHasRight(UserRight.MANAGE_RUNNERS),
|
||||
asyncMiddleware(deleteRegistrationTokenValidator),
|
||||
asyncMiddleware(deleteRegistrationToken)
|
||||
)
|
||||
|
||||
runnerRegistrationTokensRouter.get('/registration-tokens',
|
||||
authenticate,
|
||||
ensureUserHasRight(UserRight.MANAGE_RUNNERS),
|
||||
paginationValidator,
|
||||
runnerRegistrationTokensSortValidator,
|
||||
setDefaultSort,
|
||||
setDefaultPagination,
|
||||
asyncMiddleware(listRegistrationTokens)
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
runnerRegistrationTokensRouter
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function generateRegistrationToken (req: express.Request, res: express.Response) {
|
||||
logger.info('Generating new runner registration token.', lTags())
|
||||
|
||||
const registrationToken = new RunnerRegistrationTokenModel({
|
||||
registrationToken: generateRunnerRegistrationToken()
|
||||
})
|
||||
|
||||
await registrationToken.save()
|
||||
|
||||
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
|
||||
}
|
||||
|
||||
async function deleteRegistrationToken (req: express.Request, res: express.Response) {
|
||||
logger.info('Removing runner registration token.', lTags())
|
||||
|
||||
const runnerRegistrationToken = res.locals.runnerRegistrationToken
|
||||
|
||||
await runnerRegistrationToken.destroy()
|
||||
|
||||
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
|
||||
}
|
||||
|
||||
async function listRegistrationTokens (req: express.Request, res: express.Response) {
|
||||
const query: ListRunnerRegistrationTokensQuery = req.query
|
||||
|
||||
const resultList = await RunnerRegistrationTokenModel.listForApi({
|
||||
start: query.start,
|
||||
count: query.count,
|
||||
sort: query.sort
|
||||
})
|
||||
|
||||
return res.json({
|
||||
total: resultList.total,
|
||||
data: resultList.data.map(d => d.toFormattedJSON())
|
||||
})
|
||||
}
|
@ -1,10 +1,8 @@
|
||||
import Bluebird from 'bluebird'
|
||||
import express from 'express'
|
||||
import { computeResolutionsToTranscode } from '@server/helpers/ffmpeg'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger'
|
||||
import { JobQueue } from '@server/lib/job-queue'
|
||||
import { Hooks } from '@server/lib/plugins/hooks'
|
||||
import { buildTranscodingJob } from '@server/lib/video'
|
||||
import { createTranscodingJobs } from '@server/lib/transcoding/create-transcoding-job'
|
||||
import { computeResolutionsToTranscode } from '@server/lib/transcoding/transcoding-resolutions'
|
||||
import { HttpStatusCode, UserRight, VideoState, VideoTranscodingCreate } from '@shared/models'
|
||||
import { asyncMiddleware, authenticate, createTranscodingValidator, ensureUserHasRight } from '../../../middlewares'
|
||||
|
||||
@ -47,82 +45,13 @@ async function createTranscoding (req: express.Request, res: express.Response) {
|
||||
video.state = VideoState.TO_TRANSCODE
|
||||
await video.save()
|
||||
|
||||
const childrenResolutions = resolutions.filter(r => r !== maxResolution)
|
||||
|
||||
logger.info('Manually creating transcoding jobs for %s.', body.transcodingType, { childrenResolutions, maxResolution })
|
||||
|
||||
const children = await Bluebird.mapSeries(childrenResolutions, resolution => {
|
||||
if (body.transcodingType === 'hls') {
|
||||
return buildHLSJobOption({
|
||||
videoUUID: video.uuid,
|
||||
hasAudio,
|
||||
resolution,
|
||||
isMaxQuality: false
|
||||
})
|
||||
}
|
||||
|
||||
if (body.transcodingType === 'webtorrent') {
|
||||
return buildWebTorrentJobOption({
|
||||
videoUUID: video.uuid,
|
||||
hasAudio,
|
||||
resolution
|
||||
})
|
||||
}
|
||||
await createTranscodingJobs({
|
||||
video,
|
||||
resolutions,
|
||||
transcodingType: body.transcodingType,
|
||||
isNewVideo: false,
|
||||
user: null // Don't specify priority since these transcoding jobs are fired by the admin
|
||||
})
|
||||
|
||||
const parent = body.transcodingType === 'hls'
|
||||
? await buildHLSJobOption({
|
||||
videoUUID: video.uuid,
|
||||
hasAudio,
|
||||
resolution: maxResolution,
|
||||
isMaxQuality: false
|
||||
})
|
||||
: await buildWebTorrentJobOption({
|
||||
videoUUID: video.uuid,
|
||||
hasAudio,
|
||||
resolution: maxResolution
|
||||
})
|
||||
|
||||
// Porcess the last resolution after the other ones to prevent concurrency issue
|
||||
// Because low resolutions use the biggest one as ffmpeg input
|
||||
await JobQueue.Instance.createJobWithChildren(parent, children)
|
||||
|
||||
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
|
||||
}
|
||||
|
||||
function buildHLSJobOption (options: {
|
||||
videoUUID: string
|
||||
hasAudio: boolean
|
||||
resolution: number
|
||||
isMaxQuality: boolean
|
||||
}) {
|
||||
const { videoUUID, hasAudio, resolution, isMaxQuality } = options
|
||||
|
||||
return buildTranscodingJob({
|
||||
type: 'new-resolution-to-hls',
|
||||
videoUUID,
|
||||
resolution,
|
||||
hasAudio,
|
||||
copyCodecs: false,
|
||||
isNewVideo: false,
|
||||
autoDeleteWebTorrentIfNeeded: false,
|
||||
isMaxQuality
|
||||
})
|
||||
}
|
||||
|
||||
function buildWebTorrentJobOption (options: {
|
||||
videoUUID: string
|
||||
hasAudio: boolean
|
||||
resolution: number
|
||||
}) {
|
||||
const { videoUUID, hasAudio, resolution } = options
|
||||
|
||||
return buildTranscodingJob({
|
||||
type: 'new-resolution-to-webtorrent',
|
||||
videoUUID,
|
||||
isNewVideo: false,
|
||||
resolution,
|
||||
hasAudio,
|
||||
createHLSIfNeeded: false
|
||||
})
|
||||
}
|
||||
|
@ -3,28 +3,20 @@ import { move } from 'fs-extra'
|
||||
import { basename } from 'path'
|
||||
import { getResumableUploadPath } from '@server/helpers/upload'
|
||||
import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
|
||||
import { JobQueue } from '@server/lib/job-queue'
|
||||
import { generateWebTorrentVideoFilename } from '@server/lib/paths'
|
||||
import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue'
|
||||
import { Redis } from '@server/lib/redis'
|
||||
import { uploadx } from '@server/lib/uploadx'
|
||||
import {
|
||||
buildLocalVideoFromReq,
|
||||
buildMoveToObjectStorageJob,
|
||||
buildOptimizeOrMergeAudioJob,
|
||||
buildVideoThumbnailsFromReq,
|
||||
setVideoTags
|
||||
} from '@server/lib/video'
|
||||
import { buildLocalVideoFromReq, buildMoveToObjectStorageJob, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video'
|
||||
import { buildNewFile } from '@server/lib/video-file'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager'
|
||||
import { buildNextVideoState } from '@server/lib/video-state'
|
||||
import { openapiOperationDoc } from '@server/middlewares/doc'
|
||||
import { VideoSourceModel } from '@server/models/video/video-source'
|
||||
import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models'
|
||||
import { getLowercaseExtension } from '@shared/core-utils'
|
||||
import { isAudioFile, uuidToShort } from '@shared/extra-utils'
|
||||
import { HttpStatusCode, VideoCreate, VideoResolution, VideoState } from '@shared/models'
|
||||
import { uuidToShort } from '@shared/extra-utils'
|
||||
import { HttpStatusCode, VideoCreate, VideoState } from '@shared/models'
|
||||
import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
|
||||
import { createReqFiles } from '../../../helpers/express-utils'
|
||||
import { buildFileMetadata, ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '../../../helpers/ffmpeg'
|
||||
import { logger, loggerTagsFactory } from '../../../helpers/logger'
|
||||
import { MIMETYPES } from '../../../initializers/constants'
|
||||
import { sequelizeTypescript } from '../../../initializers/database'
|
||||
@ -41,7 +33,6 @@ import {
|
||||
} from '../../../middlewares'
|
||||
import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update'
|
||||
import { VideoModel } from '../../../models/video/video'
|
||||
import { VideoFileModel } from '../../../models/video/video-file'
|
||||
|
||||
const lTags = loggerTagsFactory('api', 'video')
|
||||
const auditLogger = auditLoggerFactory('videos')
|
||||
@ -148,7 +139,7 @@ async function addVideo (options: {
|
||||
video.VideoChannel = videoChannel
|
||||
video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object
|
||||
|
||||
const videoFile = await buildNewFile(videoPhysicalFile)
|
||||
const videoFile = await buildNewFile({ path: videoPhysicalFile.path, mode: 'web-video' })
|
||||
const originalFilename = videoPhysicalFile.originalname
|
||||
|
||||
// Move physical file
|
||||
@ -227,30 +218,8 @@ async function addVideo (options: {
|
||||
}
|
||||
}
|
||||
|
||||
async function buildNewFile (videoPhysicalFile: express.VideoUploadFile) {
|
||||
const videoFile = new VideoFileModel({
|
||||
extname: getLowercaseExtension(videoPhysicalFile.filename),
|
||||
size: videoPhysicalFile.size,
|
||||
videoStreamingPlaylistId: null,
|
||||
metadata: await buildFileMetadata(videoPhysicalFile.path)
|
||||
})
|
||||
|
||||
const probe = await ffprobePromise(videoPhysicalFile.path)
|
||||
|
||||
if (await isAudioFile(videoPhysicalFile.path, probe)) {
|
||||
videoFile.resolution = VideoResolution.H_NOVIDEO
|
||||
} else {
|
||||
videoFile.fps = await getVideoStreamFPS(videoPhysicalFile.path, probe)
|
||||
videoFile.resolution = (await getVideoStreamDimensionsInfo(videoPhysicalFile.path, probe)).resolution
|
||||
}
|
||||
|
||||
videoFile.filename = generateWebTorrentVideoFilename(videoFile.resolution, videoFile.extname)
|
||||
|
||||
return videoFile
|
||||
}
|
||||
|
||||
async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVideoFile, user: MUserId) {
|
||||
return JobQueue.Instance.createSequentialJobFlow(
|
||||
const jobs: (CreateJobArgument & CreateJobOptions)[] = [
|
||||
{
|
||||
type: 'manage-video-torrent' as 'manage-video-torrent',
|
||||
payload: {
|
||||
@ -274,16 +243,26 @@ async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVide
|
||||
videoUUID: video.uuid,
|
||||
isNewVideo: true
|
||||
}
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE
|
||||
? await buildMoveToObjectStorageJob({ video, previousVideoState: undefined })
|
||||
: undefined,
|
||||
if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
|
||||
jobs.push(await buildMoveToObjectStorageJob({ video, previousVideoState: undefined }))
|
||||
}
|
||||
|
||||
video.state === VideoState.TO_TRANSCODE
|
||||
? await buildOptimizeOrMergeAudioJob({ video, videoFile, user })
|
||||
: undefined
|
||||
)
|
||||
if (video.state === VideoState.TO_TRANSCODE) {
|
||||
jobs.push({
|
||||
type: 'transcoding-job-builder' as 'transcoding-job-builder',
|
||||
payload: {
|
||||
videoUUID: video.uuid,
|
||||
optimizeJob: {
|
||||
isNewVideo: true
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
return JobQueue.Instance.createSequentialJobFlow(...jobs)
|
||||
}
|
||||
|
||||
async function deleteUploadResumableCache (req: express.Request, res: express.Response, next: express.NextFunction) {
|
||||
|
@ -1,8 +1,8 @@
|
||||
import { getServerActor } from '@server/models/application/application'
|
||||
import { logger } from '@uploadx/core'
|
||||
import express from 'express'
|
||||
import { truncate } from 'lodash'
|
||||
import { SitemapStream, streamToPromise, ErrorLevel } from 'sitemap'
|
||||
import { ErrorLevel, SitemapStream, streamToPromise } from 'sitemap'
|
||||
import { logger } from '@server/helpers/logger'
|
||||
import { getServerActor } from '@server/models/application/application'
|
||||
import { buildNSFWFilter } from '../helpers/express-utils'
|
||||
import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants'
|
||||
import { asyncMiddleware } from '../middlewares'
|
||||
|
@ -1,11 +1,7 @@
|
||||
import cors from 'cors'
|
||||
import express from 'express'
|
||||
import { PassThrough, pipeline } from 'stream'
|
||||
import { logger } from '@server/helpers/logger'
|
||||
import { StreamReplacer } from '@server/helpers/stream-replacer'
|
||||
import { OBJECT_STORAGE_PROXY_PATHS } from '@server/initializers/constants'
|
||||
import { injectQueryToPlaylistUrls } from '@server/lib/hls'
|
||||
import { getHLSFileReadStream, getWebTorrentFileReadStream } from '@server/lib/object-storage'
|
||||
import { proxifyHLS, proxifyWebTorrentFile } from '@server/lib/object-storage'
|
||||
import {
|
||||
asyncMiddleware,
|
||||
ensureCanAccessPrivateVideoHLSFiles,
|
||||
@ -13,9 +9,7 @@ import {
|
||||
ensurePrivateObjectStorageProxyIsEnabled,
|
||||
optionalAuthenticate
|
||||
} from '@server/middlewares'
|
||||
import { HttpStatusCode } from '@shared/models'
|
||||
import { buildReinjectVideoFileTokenQuery, doReinjectVideoFileToken } from './shared/m3u8-playlist'
|
||||
import { GetObjectCommandOutput } from '@aws-sdk/client-s3'
|
||||
import { doReinjectVideoFileToken } from './shared/m3u8-playlist'
|
||||
|
||||
const objectStorageProxyRouter = express.Router()
|
||||
|
||||
@ -25,14 +19,14 @@ objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEBSEED + ':file
|
||||
ensurePrivateObjectStorageProxyIsEnabled,
|
||||
optionalAuthenticate,
|
||||
asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles),
|
||||
asyncMiddleware(proxifyWebTorrent)
|
||||
asyncMiddleware(proxifyWebTorrentController)
|
||||
)
|
||||
|
||||
objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:filename',
|
||||
ensurePrivateObjectStorageProxyIsEnabled,
|
||||
optionalAuthenticate,
|
||||
asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles),
|
||||
asyncMiddleware(proxifyHLS)
|
||||
asyncMiddleware(proxifyHLSController)
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@ -41,76 +35,25 @@ export {
|
||||
objectStorageProxyRouter
|
||||
}
|
||||
|
||||
async function proxifyWebTorrent (req: express.Request, res: express.Response) {
|
||||
function proxifyWebTorrentController (req: express.Request, res: express.Response) {
|
||||
const filename = req.params.filename
|
||||
|
||||
logger.debug('Proxifying WebTorrent file %s from object storage.', filename)
|
||||
|
||||
try {
|
||||
const { response: s3Response, stream } = await getWebTorrentFileReadStream({
|
||||
filename,
|
||||
rangeHeader: req.header('range')
|
||||
})
|
||||
|
||||
setS3Headers(res, s3Response)
|
||||
|
||||
return stream.pipe(res)
|
||||
} catch (err) {
|
||||
return handleObjectStorageFailure(res, err)
|
||||
}
|
||||
return proxifyWebTorrentFile({ req, res, filename })
|
||||
}
|
||||
|
||||
async function proxifyHLS (req: express.Request, res: express.Response) {
|
||||
function proxifyHLSController (req: express.Request, res: express.Response) {
|
||||
const playlist = res.locals.videoStreamingPlaylist
|
||||
const video = res.locals.onlyVideo
|
||||
const filename = req.params.filename
|
||||
|
||||
logger.debug('Proxifying HLS file %s from object storage.', filename)
|
||||
const reinjectVideoFileToken = filename.endsWith('.m3u8') && doReinjectVideoFileToken(req)
|
||||
|
||||
try {
|
||||
const { response: s3Response, stream } = await getHLSFileReadStream({
|
||||
playlist: playlist.withVideo(video),
|
||||
filename,
|
||||
rangeHeader: req.header('range')
|
||||
})
|
||||
|
||||
setS3Headers(res, s3Response)
|
||||
|
||||
const streamReplacer = filename.endsWith('.m3u8') && doReinjectVideoFileToken(req)
|
||||
? new StreamReplacer(line => injectQueryToPlaylistUrls(line, buildReinjectVideoFileTokenQuery(req, filename.endsWith('master.m3u8'))))
|
||||
: new PassThrough()
|
||||
|
||||
return pipeline(
|
||||
stream,
|
||||
streamReplacer,
|
||||
res,
|
||||
err => {
|
||||
if (!err) return
|
||||
|
||||
handleObjectStorageFailure(res, err)
|
||||
}
|
||||
)
|
||||
} catch (err) {
|
||||
return handleObjectStorageFailure(res, err)
|
||||
}
|
||||
}
|
||||
|
||||
function handleObjectStorageFailure (res: express.Response, err: Error) {
|
||||
if (err.name === 'NoSuchKey') {
|
||||
logger.debug('Could not find key in object storage to proxify private HLS video file.', { err })
|
||||
return res.sendStatus(HttpStatusCode.NOT_FOUND_404)
|
||||
}
|
||||
|
||||
return res.fail({
|
||||
status: HttpStatusCode.INTERNAL_SERVER_ERROR_500,
|
||||
message: err.message,
|
||||
type: err.name
|
||||
return proxifyHLS({
|
||||
req,
|
||||
res,
|
||||
playlist,
|
||||
video,
|
||||
filename,
|
||||
reinjectVideoFileToken
|
||||
})
|
||||
}
|
||||
|
||||
function setS3Headers (res: express.Response, s3Response: GetObjectCommandOutput) {
|
||||
if (s3Response.$metadata.httpStatusCode === HttpStatusCode.PARTIAL_CONTENT_206) {
|
||||
res.setHeader('Content-Range', s3Response.ContentRange)
|
||||
res.status(HttpStatusCode.PARTIAL_CONTENT_206)
|
||||
}
|
||||
}
|
||||
|
@ -11,6 +11,7 @@ import { truncate } from 'lodash'
|
||||
import { pipeline } from 'stream'
|
||||
import { URL } from 'url'
|
||||
import { promisify } from 'util'
|
||||
import { promisify1, promisify2, promisify3 } from '@shared/core-utils'
|
||||
|
||||
const objectConverter = (oldObject: any, keyConverter: (e: string) => string, valueConverter: (e: any) => any) => {
|
||||
if (!oldObject || typeof oldObject !== 'object') {
|
||||
@ -229,18 +230,6 @@ function execShell (command: string, options?: ExecOptions) {
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function isOdd (num: number) {
|
||||
return (num % 2) !== 0
|
||||
}
|
||||
|
||||
function toEven (num: number) {
|
||||
if (isOdd(num)) return num + 1
|
||||
|
||||
return num
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function generateRSAKeyPairPromise (size: number) {
|
||||
return new Promise<{ publicKey: string, privateKey: string }>((res, rej) => {
|
||||
const options: RSAKeyPairOptions<'pem', 'pem'> = {
|
||||
@ -286,40 +275,6 @@ function generateED25519KeyPairPromise () {
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function promisify0<A> (func: (cb: (err: any, result: A) => void) => void): () => Promise<A> {
|
||||
return function promisified (): Promise<A> {
|
||||
return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
|
||||
func.apply(null, [ (err: any, res: A) => err ? reject(err) : resolve(res) ])
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Thanks to https://gist.github.com/kumasento/617daa7e46f13ecdd9b2
|
||||
function promisify1<T, A> (func: (arg: T, cb: (err: any, result: A) => void) => void): (arg: T) => Promise<A> {
|
||||
return function promisified (arg: T): Promise<A> {
|
||||
return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
|
||||
func.apply(null, [ arg, (err: any, res: A) => err ? reject(err) : resolve(res) ])
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function promisify2<T, U, A> (func: (arg1: T, arg2: U, cb: (err: any, result: A) => void) => void): (arg1: T, arg2: U) => Promise<A> {
|
||||
return function promisified (arg1: T, arg2: U): Promise<A> {
|
||||
return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
|
||||
func.apply(null, [ arg1, arg2, (err: any, res: A) => err ? reject(err) : resolve(res) ])
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// eslint-disable-next-line max-len
|
||||
function promisify3<T, U, V, A> (func: (arg1: T, arg2: U, arg3: V, cb: (err: any, result: A) => void) => void): (arg1: T, arg2: U, arg3: V) => Promise<A> {
|
||||
return function promisified (arg1: T, arg2: U, arg3: V): Promise<A> {
|
||||
return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
|
||||
func.apply(null, [ arg1, arg2, arg3, (err: any, res: A) => err ? reject(err) : resolve(res) ])
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const randomBytesPromise = promisify1<number, Buffer>(randomBytes)
|
||||
const scryptPromise = promisify3<string, string, number, Buffer>(scrypt)
|
||||
const execPromise2 = promisify2<string, any, string>(exec)
|
||||
@ -345,10 +300,6 @@ export {
|
||||
pageToStartAndCount,
|
||||
peertubeTruncate,
|
||||
|
||||
promisify0,
|
||||
promisify1,
|
||||
promisify2,
|
||||
|
||||
scryptPromise,
|
||||
|
||||
randomBytesPromise,
|
||||
@ -360,8 +311,5 @@ export {
|
||||
execPromise,
|
||||
pipelinePromise,
|
||||
|
||||
parseSemVersion,
|
||||
|
||||
isOdd,
|
||||
toEven
|
||||
parseSemVersion
|
||||
}
|
||||
|
@ -15,6 +15,10 @@ function isSafePath (p: string) {
|
||||
})
|
||||
}
|
||||
|
||||
function isSafeFilename (filename: string, extension: string) {
|
||||
return typeof filename === 'string' && !!filename.match(new RegExp(`^[a-z0-9-]+\\.${extension}$`))
|
||||
}
|
||||
|
||||
function isSafePeerTubeFilenameWithoutExtension (filename: string) {
|
||||
return filename.match(/^[a-z0-9-]+$/)
|
||||
}
|
||||
@ -177,5 +181,6 @@ export {
|
||||
toIntArray,
|
||||
isFileValid,
|
||||
isSafePeerTubeFilenameWithoutExtension,
|
||||
isSafeFilename,
|
||||
checkMimetypeRegex
|
||||
}
|
||||
|
166
server/helpers/custom-validators/runners/jobs.ts
Normal file
166
server/helpers/custom-validators/runners/jobs.ts
Normal file
@ -0,0 +1,166 @@
|
||||
import { UploadFilesForCheck } from 'express'
|
||||
import validator from 'validator'
|
||||
import { CONSTRAINTS_FIELDS } from '@server/initializers/constants'
|
||||
import {
|
||||
LiveRTMPHLSTranscodingSuccess,
|
||||
RunnerJobSuccessPayload,
|
||||
RunnerJobType,
|
||||
RunnerJobUpdatePayload,
|
||||
VODAudioMergeTranscodingSuccess,
|
||||
VODHLSTranscodingSuccess,
|
||||
VODWebVideoTranscodingSuccess
|
||||
} from '@shared/models'
|
||||
import { exists, isFileValid, isSafeFilename } from '../misc'
|
||||
|
||||
const RUNNER_JOBS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.RUNNER_JOBS
|
||||
|
||||
const runnerJobTypes = new Set([ 'vod-hls-transcoding', 'vod-web-video-transcoding', 'vod-audio-merge-transcoding' ])
|
||||
function isRunnerJobTypeValid (value: RunnerJobType) {
|
||||
return runnerJobTypes.has(value)
|
||||
}
|
||||
|
||||
function isRunnerJobSuccessPayloadValid (value: RunnerJobSuccessPayload, type: RunnerJobType, files: UploadFilesForCheck) {
|
||||
return isRunnerJobVODWebVideoResultPayloadValid(value as VODWebVideoTranscodingSuccess, type, files) ||
|
||||
isRunnerJobVODHLSResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) ||
|
||||
isRunnerJobVODAudioMergeResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) ||
|
||||
isRunnerJobLiveRTMPHLSResultPayloadValid(value as LiveRTMPHLSTranscodingSuccess, type)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function isRunnerJobProgressValid (value: string) {
|
||||
return validator.isInt(value + '', RUNNER_JOBS_CONSTRAINTS_FIELDS.PROGRESS)
|
||||
}
|
||||
|
||||
function isRunnerJobUpdatePayloadValid (value: RunnerJobUpdatePayload, type: RunnerJobType, files: UploadFilesForCheck) {
|
||||
return isRunnerJobVODWebVideoUpdatePayloadValid(value, type, files) ||
|
||||
isRunnerJobVODHLSUpdatePayloadValid(value, type, files) ||
|
||||
isRunnerJobVODAudioMergeUpdatePayloadValid(value, type, files) ||
|
||||
isRunnerJobLiveRTMPHLSUpdatePayloadValid(value, type, files)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function isRunnerJobTokenValid (value: string) {
|
||||
return exists(value) && validator.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.TOKEN)
|
||||
}
|
||||
|
||||
function isRunnerJobAbortReasonValid (value: string) {
|
||||
return validator.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.REASON)
|
||||
}
|
||||
|
||||
function isRunnerJobErrorMessageValid (value: string) {
|
||||
return validator.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.ERROR_MESSAGE)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
isRunnerJobTypeValid,
|
||||
isRunnerJobSuccessPayloadValid,
|
||||
isRunnerJobUpdatePayloadValid,
|
||||
isRunnerJobTokenValid,
|
||||
isRunnerJobErrorMessageValid,
|
||||
isRunnerJobProgressValid,
|
||||
isRunnerJobAbortReasonValid
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function isRunnerJobVODWebVideoResultPayloadValid (
|
||||
_value: VODWebVideoTranscodingSuccess,
|
||||
type: RunnerJobType,
|
||||
files: UploadFilesForCheck
|
||||
) {
|
||||
return type === 'vod-web-video-transcoding' &&
|
||||
isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null })
|
||||
}
|
||||
|
||||
function isRunnerJobVODHLSResultPayloadValid (
|
||||
_value: VODHLSTranscodingSuccess,
|
||||
type: RunnerJobType,
|
||||
files: UploadFilesForCheck
|
||||
) {
|
||||
return type === 'vod-hls-transcoding' &&
|
||||
isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) &&
|
||||
isFileValid({ files, field: 'payload[resolutionPlaylistFile]', mimeTypeRegex: null, maxSize: null })
|
||||
}
|
||||
|
||||
function isRunnerJobVODAudioMergeResultPayloadValid (
|
||||
_value: VODAudioMergeTranscodingSuccess,
|
||||
type: RunnerJobType,
|
||||
files: UploadFilesForCheck
|
||||
) {
|
||||
return type === 'vod-audio-merge-transcoding' &&
|
||||
isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null })
|
||||
}
|
||||
|
||||
function isRunnerJobLiveRTMPHLSResultPayloadValid (
|
||||
value: LiveRTMPHLSTranscodingSuccess,
|
||||
type: RunnerJobType
|
||||
) {
|
||||
return type === 'live-rtmp-hls-transcoding' && (!value || (typeof value === 'object' && Object.keys(value).length === 0))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function isRunnerJobVODWebVideoUpdatePayloadValid (
|
||||
value: RunnerJobUpdatePayload,
|
||||
type: RunnerJobType,
|
||||
_files: UploadFilesForCheck
|
||||
) {
|
||||
return type === 'vod-web-video-transcoding' &&
|
||||
(!value || (typeof value === 'object' && Object.keys(value).length === 0))
|
||||
}
|
||||
|
||||
function isRunnerJobVODHLSUpdatePayloadValid (
|
||||
value: RunnerJobUpdatePayload,
|
||||
type: RunnerJobType,
|
||||
_files: UploadFilesForCheck
|
||||
) {
|
||||
return type === 'vod-hls-transcoding' &&
|
||||
(!value || (typeof value === 'object' && Object.keys(value).length === 0))
|
||||
}
|
||||
|
||||
function isRunnerJobVODAudioMergeUpdatePayloadValid (
|
||||
value: RunnerJobUpdatePayload,
|
||||
type: RunnerJobType,
|
||||
_files: UploadFilesForCheck
|
||||
) {
|
||||
return type === 'vod-audio-merge-transcoding' &&
|
||||
(!value || (typeof value === 'object' && Object.keys(value).length === 0))
|
||||
}
|
||||
|
||||
function isRunnerJobLiveRTMPHLSUpdatePayloadValid (
|
||||
value: RunnerJobUpdatePayload,
|
||||
type: RunnerJobType,
|
||||
files: UploadFilesForCheck
|
||||
) {
|
||||
let result = type === 'live-rtmp-hls-transcoding' && !!value && !!files
|
||||
|
||||
result &&= isFileValid({ files, field: 'payload[masterPlaylistFile]', mimeTypeRegex: null, maxSize: null, optional: true })
|
||||
|
||||
result &&= isFileValid({
|
||||
files,
|
||||
field: 'payload[resolutionPlaylistFile]',
|
||||
mimeTypeRegex: null,
|
||||
maxSize: null,
|
||||
optional: !value.resolutionPlaylistFilename
|
||||
})
|
||||
|
||||
if (files['payload[resolutionPlaylistFile]']) {
|
||||
result &&= isSafeFilename(value.resolutionPlaylistFilename, 'm3u8')
|
||||
}
|
||||
|
||||
return result &&
|
||||
isSafeFilename(value.videoChunkFilename, 'ts') &&
|
||||
(
|
||||
(
|
||||
value.type === 'remove-chunk'
|
||||
) ||
|
||||
(
|
||||
value.type === 'add-chunk' &&
|
||||
isFileValid({ files, field: 'payload[videoChunkFile]', mimeTypeRegex: null, maxSize: null })
|
||||
)
|
||||
)
|
||||
}
|
30
server/helpers/custom-validators/runners/runners.ts
Normal file
30
server/helpers/custom-validators/runners/runners.ts
Normal file
@ -0,0 +1,30 @@
|
||||
import validator from 'validator'
|
||||
import { CONSTRAINTS_FIELDS } from '@server/initializers/constants'
|
||||
import { exists } from '../misc'
|
||||
|
||||
const RUNNERS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.RUNNERS
|
||||
|
||||
function isRunnerRegistrationTokenValid (value: string) {
|
||||
return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.TOKEN)
|
||||
}
|
||||
|
||||
function isRunnerTokenValid (value: string) {
|
||||
return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.TOKEN)
|
||||
}
|
||||
|
||||
function isRunnerNameValid (value: string) {
|
||||
return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.NAME)
|
||||
}
|
||||
|
||||
function isRunnerDescriptionValid (value: string) {
|
||||
return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.DESCRIPTION)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
isRunnerRegistrationTokenValid,
|
||||
isRunnerTokenValid,
|
||||
isRunnerNameValid,
|
||||
isRunnerDescriptionValid
|
||||
}
|
16
server/helpers/debounce.ts
Normal file
16
server/helpers/debounce.ts
Normal file
@ -0,0 +1,16 @@
|
||||
export function Debounce (config: { timeoutMS: number }) {
|
||||
let timeoutRef: NodeJS.Timeout
|
||||
|
||||
return function (_target, _key, descriptor: PropertyDescriptor) {
|
||||
const original = descriptor.value
|
||||
|
||||
descriptor.value = function (...args: any[]) {
|
||||
clearTimeout(timeoutRef)
|
||||
|
||||
timeoutRef = setTimeout(() => {
|
||||
original.apply(this, args)
|
||||
|
||||
}, config.timeoutMS)
|
||||
}
|
||||
}
|
||||
}
|
64
server/helpers/ffmpeg/codecs.ts
Normal file
64
server/helpers/ffmpeg/codecs.ts
Normal file
@ -0,0 +1,64 @@
|
||||
import { FfprobeData } from 'fluent-ffmpeg'
|
||||
import { getAudioStream, getVideoStream } from '@shared/ffmpeg'
|
||||
import { logger } from '../logger'
|
||||
import { forceNumber } from '@shared/core-utils'
|
||||
|
||||
export async function getVideoStreamCodec (path: string) {
|
||||
const videoStream = await getVideoStream(path)
|
||||
if (!videoStream) return ''
|
||||
|
||||
const videoCodec = videoStream.codec_tag_string
|
||||
|
||||
if (videoCodec === 'vp09') return 'vp09.00.50.08'
|
||||
if (videoCodec === 'hev1') return 'hev1.1.6.L93.B0'
|
||||
|
||||
const baseProfileMatrix = {
|
||||
avc1: {
|
||||
High: '6400',
|
||||
Main: '4D40',
|
||||
Baseline: '42E0'
|
||||
},
|
||||
av01: {
|
||||
High: '1',
|
||||
Main: '0',
|
||||
Professional: '2'
|
||||
}
|
||||
}
|
||||
|
||||
let baseProfile = baseProfileMatrix[videoCodec][videoStream.profile]
|
||||
if (!baseProfile) {
|
||||
logger.warn('Cannot get video profile codec of %s.', path, { videoStream })
|
||||
baseProfile = baseProfileMatrix[videoCodec]['High'] // Fallback
|
||||
}
|
||||
|
||||
if (videoCodec === 'av01') {
|
||||
let level = videoStream.level.toString()
|
||||
if (level.length === 1) level = `0${level}`
|
||||
|
||||
// Guess the tier indicator and bit depth
|
||||
return `${videoCodec}.${baseProfile}.${level}M.08`
|
||||
}
|
||||
|
||||
let level = forceNumber(videoStream.level).toString(16)
|
||||
if (level.length === 1) level = `0${level}`
|
||||
|
||||
// Default, h264 codec
|
||||
return `${videoCodec}.${baseProfile}${level}`
|
||||
}
|
||||
|
||||
export async function getAudioStreamCodec (path: string, existingProbe?: FfprobeData) {
|
||||
const { audioStream } = await getAudioStream(path, existingProbe)
|
||||
|
||||
if (!audioStream) return ''
|
||||
|
||||
const audioCodecName = audioStream.codec_name
|
||||
|
||||
if (audioCodecName === 'opus') return 'opus'
|
||||
if (audioCodecName === 'vorbis') return 'vorbis'
|
||||
if (audioCodecName === 'aac') return 'mp4a.40.2'
|
||||
if (audioCodecName === 'mp3') return 'mp4a.40.34'
|
||||
|
||||
logger.warn('Cannot get audio codec of %s.', path, { audioStream })
|
||||
|
||||
return 'mp4a.40.2' // Fallback
|
||||
}
|
@ -1,114 +0,0 @@
|
||||
import { Job } from 'bullmq'
|
||||
import ffmpeg, { FfmpegCommand } from 'fluent-ffmpeg'
|
||||
import { execPromise } from '@server/helpers/core-utils'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger'
|
||||
import { CONFIG } from '@server/initializers/config'
|
||||
import { FFMPEG_NICE } from '@server/initializers/constants'
|
||||
import { EncoderOptions } from '@shared/models'
|
||||
|
||||
const lTags = loggerTagsFactory('ffmpeg')
|
||||
|
||||
type StreamType = 'audio' | 'video'
|
||||
|
||||
function getFFmpeg (input: string, type: 'live' | 'vod') {
|
||||
// We set cwd explicitly because ffmpeg appears to create temporary files when trancoding which fails in read-only file systems
|
||||
const command = ffmpeg(input, {
|
||||
niceness: type === 'live' ? FFMPEG_NICE.LIVE : FFMPEG_NICE.VOD,
|
||||
cwd: CONFIG.STORAGE.TMP_DIR
|
||||
})
|
||||
|
||||
const threads = type === 'live'
|
||||
? CONFIG.LIVE.TRANSCODING.THREADS
|
||||
: CONFIG.TRANSCODING.THREADS
|
||||
|
||||
if (threads > 0) {
|
||||
// If we don't set any threads ffmpeg will chose automatically
|
||||
command.outputOption('-threads ' + threads)
|
||||
}
|
||||
|
||||
return command
|
||||
}
|
||||
|
||||
function getFFmpegVersion () {
|
||||
return new Promise<string>((res, rej) => {
|
||||
(ffmpeg() as any)._getFfmpegPath((err, ffmpegPath) => {
|
||||
if (err) return rej(err)
|
||||
if (!ffmpegPath) return rej(new Error('Could not find ffmpeg path'))
|
||||
|
||||
return execPromise(`${ffmpegPath} -version`)
|
||||
.then(stdout => {
|
||||
const parsed = stdout.match(/ffmpeg version .?(\d+\.\d+(\.\d+)?)/)
|
||||
if (!parsed?.[1]) return rej(new Error(`Could not find ffmpeg version in ${stdout}`))
|
||||
|
||||
// Fix ffmpeg version that does not include patch version (4.4 for example)
|
||||
let version = parsed[1]
|
||||
if (version.match(/^\d+\.\d+$/)) {
|
||||
version += '.0'
|
||||
}
|
||||
|
||||
return res(version)
|
||||
})
|
||||
.catch(err => rej(err))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async function runCommand (options: {
|
||||
command: FfmpegCommand
|
||||
silent?: boolean // false by default
|
||||
job?: Job
|
||||
}) {
|
||||
const { command, silent = false, job } = options
|
||||
|
||||
return new Promise<void>((res, rej) => {
|
||||
let shellCommand: string
|
||||
|
||||
command.on('start', cmdline => { shellCommand = cmdline })
|
||||
|
||||
command.on('error', (err, stdout, stderr) => {
|
||||
if (silent !== true) logger.error('Error in ffmpeg.', { stdout, stderr, shellCommand, ...lTags() })
|
||||
|
||||
rej(err)
|
||||
})
|
||||
|
||||
command.on('end', (stdout, stderr) => {
|
||||
logger.debug('FFmpeg command ended.', { stdout, stderr, shellCommand, ...lTags() })
|
||||
|
||||
res()
|
||||
})
|
||||
|
||||
if (job) {
|
||||
command.on('progress', progress => {
|
||||
if (!progress.percent) return
|
||||
|
||||
job.updateProgress(Math.round(progress.percent))
|
||||
.catch(err => logger.warn('Cannot set ffmpeg job progress.', { err, ...lTags() }))
|
||||
})
|
||||
}
|
||||
|
||||
command.run()
|
||||
})
|
||||
}
|
||||
|
||||
function buildStreamSuffix (base: string, streamNum?: number) {
|
||||
if (streamNum !== undefined) {
|
||||
return `${base}:${streamNum}`
|
||||
}
|
||||
|
||||
return base
|
||||
}
|
||||
|
||||
function getScaleFilter (options: EncoderOptions): string {
|
||||
if (options.scaleFilter) return options.scaleFilter.name
|
||||
|
||||
return 'scale'
|
||||
}
|
||||
|
||||
export {
|
||||
getFFmpeg,
|
||||
getFFmpegVersion,
|
||||
runCommand,
|
||||
StreamType,
|
||||
buildStreamSuffix,
|
||||
getScaleFilter
|
||||
}
|
@ -1,258 +0,0 @@
|
||||
import { FilterSpecification } from 'fluent-ffmpeg'
|
||||
import { VIDEO_FILTERS } from '@server/initializers/constants'
|
||||
import { AvailableEncoders } from '@shared/models'
|
||||
import { logger, loggerTagsFactory } from '../logger'
|
||||
import { getFFmpeg, runCommand } from './ffmpeg-commons'
|
||||
import { presetVOD } from './ffmpeg-presets'
|
||||
import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS, hasAudioStream } from './ffprobe-utils'
|
||||
|
||||
const lTags = loggerTagsFactory('ffmpeg')
|
||||
|
||||
async function cutVideo (options: {
|
||||
inputPath: string
|
||||
outputPath: string
|
||||
start?: number
|
||||
end?: number
|
||||
|
||||
availableEncoders: AvailableEncoders
|
||||
profile: string
|
||||
}) {
|
||||
const { inputPath, outputPath, availableEncoders, profile } = options
|
||||
|
||||
logger.debug('Will cut the video.', { options, ...lTags() })
|
||||
|
||||
const mainProbe = await ffprobePromise(inputPath)
|
||||
const fps = await getVideoStreamFPS(inputPath, mainProbe)
|
||||
const { resolution } = await getVideoStreamDimensionsInfo(inputPath, mainProbe)
|
||||
|
||||
let command = getFFmpeg(inputPath, 'vod')
|
||||
.output(outputPath)
|
||||
|
||||
command = await presetVOD({
|
||||
command,
|
||||
input: inputPath,
|
||||
availableEncoders,
|
||||
profile,
|
||||
resolution,
|
||||
fps,
|
||||
canCopyAudio: false,
|
||||
canCopyVideo: false
|
||||
})
|
||||
|
||||
if (options.start) {
|
||||
command.outputOption('-ss ' + options.start)
|
||||
}
|
||||
|
||||
if (options.end) {
|
||||
command.outputOption('-to ' + options.end)
|
||||
}
|
||||
|
||||
await runCommand({ command })
|
||||
}
|
||||
|
||||
async function addWatermark (options: {
|
||||
inputPath: string
|
||||
watermarkPath: string
|
||||
outputPath: string
|
||||
|
||||
availableEncoders: AvailableEncoders
|
||||
profile: string
|
||||
}) {
|
||||
const { watermarkPath, inputPath, outputPath, availableEncoders, profile } = options
|
||||
|
||||
logger.debug('Will add watermark to the video.', { options, ...lTags() })
|
||||
|
||||
const videoProbe = await ffprobePromise(inputPath)
|
||||
const fps = await getVideoStreamFPS(inputPath, videoProbe)
|
||||
const { resolution } = await getVideoStreamDimensionsInfo(inputPath, videoProbe)
|
||||
|
||||
let command = getFFmpeg(inputPath, 'vod')
|
||||
.output(outputPath)
|
||||
command.input(watermarkPath)
|
||||
|
||||
command = await presetVOD({
|
||||
command,
|
||||
input: inputPath,
|
||||
availableEncoders,
|
||||
profile,
|
||||
resolution,
|
||||
fps,
|
||||
canCopyAudio: true,
|
||||
canCopyVideo: false
|
||||
})
|
||||
|
||||
const complexFilter: FilterSpecification[] = [
|
||||
// Scale watermark
|
||||
{
|
||||
inputs: [ '[1]', '[0]' ],
|
||||
filter: 'scale2ref',
|
||||
options: {
|
||||
w: 'oh*mdar',
|
||||
h: `ih*${VIDEO_FILTERS.WATERMARK.SIZE_RATIO}`
|
||||
},
|
||||
outputs: [ '[watermark]', '[video]' ]
|
||||
},
|
||||
|
||||
{
|
||||
inputs: [ '[video]', '[watermark]' ],
|
||||
filter: 'overlay',
|
||||
options: {
|
||||
x: `main_w - overlay_w - (main_h * ${VIDEO_FILTERS.WATERMARK.HORIZONTAL_MARGIN_RATIO})`,
|
||||
y: `main_h * ${VIDEO_FILTERS.WATERMARK.VERTICAL_MARGIN_RATIO}`
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
command.complexFilter(complexFilter)
|
||||
|
||||
await runCommand({ command })
|
||||
}
|
||||
|
||||
async function addIntroOutro (options: {
|
||||
inputPath: string
|
||||
introOutroPath: string
|
||||
outputPath: string
|
||||
type: 'intro' | 'outro'
|
||||
|
||||
availableEncoders: AvailableEncoders
|
||||
profile: string
|
||||
}) {
|
||||
const { introOutroPath, inputPath, outputPath, availableEncoders, profile, type } = options
|
||||
|
||||
logger.debug('Will add intro/outro to the video.', { options, ...lTags() })
|
||||
|
||||
const mainProbe = await ffprobePromise(inputPath)
|
||||
const fps = await getVideoStreamFPS(inputPath, mainProbe)
|
||||
const { resolution } = await getVideoStreamDimensionsInfo(inputPath, mainProbe)
|
||||
const mainHasAudio = await hasAudioStream(inputPath, mainProbe)
|
||||
|
||||
const introOutroProbe = await ffprobePromise(introOutroPath)
|
||||
const introOutroHasAudio = await hasAudioStream(introOutroPath, introOutroProbe)
|
||||
|
||||
let command = getFFmpeg(inputPath, 'vod')
|
||||
.output(outputPath)
|
||||
|
||||
command.input(introOutroPath)
|
||||
|
||||
if (!introOutroHasAudio && mainHasAudio) {
|
||||
const duration = await getVideoStreamDuration(introOutroPath, introOutroProbe)
|
||||
|
||||
command.input('anullsrc')
|
||||
command.withInputFormat('lavfi')
|
||||
command.withInputOption('-t ' + duration)
|
||||
}
|
||||
|
||||
command = await presetVOD({
|
||||
command,
|
||||
input: inputPath,
|
||||
availableEncoders,
|
||||
profile,
|
||||
resolution,
|
||||
fps,
|
||||
canCopyAudio: false,
|
||||
canCopyVideo: false
|
||||
})
|
||||
|
||||
// Add black background to correctly scale intro/outro with padding
|
||||
const complexFilter: FilterSpecification[] = [
|
||||
{
|
||||
inputs: [ '1', '0' ],
|
||||
filter: 'scale2ref',
|
||||
options: {
|
||||
w: 'iw',
|
||||
h: `ih`
|
||||
},
|
||||
outputs: [ 'intro-outro', 'main' ]
|
||||
},
|
||||
{
|
||||
inputs: [ 'intro-outro', 'main' ],
|
||||
filter: 'scale2ref',
|
||||
options: {
|
||||
w: 'iw',
|
||||
h: `ih`
|
||||
},
|
||||
outputs: [ 'to-scale', 'main' ]
|
||||
},
|
||||
{
|
||||
inputs: 'to-scale',
|
||||
filter: 'drawbox',
|
||||
options: {
|
||||
t: 'fill'
|
||||
},
|
||||
outputs: [ 'to-scale-bg' ]
|
||||
},
|
||||
{
|
||||
inputs: [ '1', 'to-scale-bg' ],
|
||||
filter: 'scale2ref',
|
||||
options: {
|
||||
w: 'iw',
|
||||
h: 'ih',
|
||||
force_original_aspect_ratio: 'decrease',
|
||||
flags: 'spline'
|
||||
},
|
||||
outputs: [ 'to-scale', 'to-scale-bg' ]
|
||||
},
|
||||
{
|
||||
inputs: [ 'to-scale-bg', 'to-scale' ],
|
||||
filter: 'overlay',
|
||||
options: {
|
||||
x: '(main_w - overlay_w)/2',
|
||||
y: '(main_h - overlay_h)/2'
|
||||
},
|
||||
outputs: 'intro-outro-resized'
|
||||
}
|
||||
]
|
||||
|
||||
const concatFilter = {
|
||||
inputs: [],
|
||||
filter: 'concat',
|
||||
options: {
|
||||
n: 2,
|
||||
v: 1,
|
||||
unsafe: 1
|
||||
},
|
||||
outputs: [ 'v' ]
|
||||
}
|
||||
|
||||
const introOutroFilterInputs = [ 'intro-outro-resized' ]
|
||||
const mainFilterInputs = [ 'main' ]
|
||||
|
||||
if (mainHasAudio) {
|
||||
mainFilterInputs.push('0:a')
|
||||
|
||||
if (introOutroHasAudio) {
|
||||
introOutroFilterInputs.push('1:a')
|
||||
} else {
|
||||
// Silent input
|
||||
introOutroFilterInputs.push('2:a')
|
||||
}
|
||||
}
|
||||
|
||||
if (type === 'intro') {
|
||||
concatFilter.inputs = [ ...introOutroFilterInputs, ...mainFilterInputs ]
|
||||
} else {
|
||||
concatFilter.inputs = [ ...mainFilterInputs, ...introOutroFilterInputs ]
|
||||
}
|
||||
|
||||
if (mainHasAudio) {
|
||||
concatFilter.options['a'] = 1
|
||||
concatFilter.outputs.push('a')
|
||||
|
||||
command.outputOption('-map [a]')
|
||||
}
|
||||
|
||||
command.outputOption('-map [v]')
|
||||
|
||||
complexFilter.push(concatFilter)
|
||||
command.complexFilter(complexFilter)
|
||||
|
||||
await runCommand({ command })
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
cutVideo,
|
||||
addIntroOutro,
|
||||
addWatermark
|
||||
}
|
@ -1,116 +0,0 @@
|
||||
import { getAvailableEncoders } from 'fluent-ffmpeg'
|
||||
import { pick } from '@shared/core-utils'
|
||||
import { AvailableEncoders, EncoderOptionsBuilder, EncoderOptionsBuilderParams, EncoderProfile } from '@shared/models'
|
||||
import { promisify0 } from '../core-utils'
|
||||
import { logger, loggerTagsFactory } from '../logger'
|
||||
|
||||
const lTags = loggerTagsFactory('ffmpeg')
|
||||
|
||||
// Detect supported encoders by ffmpeg
|
||||
let supportedEncoders: Map<string, boolean>
|
||||
async function checkFFmpegEncoders (peertubeAvailableEncoders: AvailableEncoders): Promise<Map<string, boolean>> {
|
||||
if (supportedEncoders !== undefined) {
|
||||
return supportedEncoders
|
||||
}
|
||||
|
||||
const getAvailableEncodersPromise = promisify0(getAvailableEncoders)
|
||||
const availableFFmpegEncoders = await getAvailableEncodersPromise()
|
||||
|
||||
const searchEncoders = new Set<string>()
|
||||
for (const type of [ 'live', 'vod' ]) {
|
||||
for (const streamType of [ 'audio', 'video' ]) {
|
||||
for (const encoder of peertubeAvailableEncoders.encodersToTry[type][streamType]) {
|
||||
searchEncoders.add(encoder)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
supportedEncoders = new Map<string, boolean>()
|
||||
|
||||
for (const searchEncoder of searchEncoders) {
|
||||
supportedEncoders.set(searchEncoder, availableFFmpegEncoders[searchEncoder] !== undefined)
|
||||
}
|
||||
|
||||
logger.info('Built supported ffmpeg encoders.', { supportedEncoders, searchEncoders, ...lTags() })
|
||||
|
||||
return supportedEncoders
|
||||
}
|
||||
|
||||
function resetSupportedEncoders () {
|
||||
supportedEncoders = undefined
|
||||
}
|
||||
|
||||
// Run encoder builder depending on available encoders
|
||||
// Try encoders by priority: if the encoder is available, run the chosen profile or fallback to the default one
|
||||
// If the default one does not exist, check the next encoder
|
||||
async function getEncoderBuilderResult (options: EncoderOptionsBuilderParams & {
|
||||
streamType: 'video' | 'audio'
|
||||
input: string
|
||||
|
||||
availableEncoders: AvailableEncoders
|
||||
profile: string
|
||||
|
||||
videoType: 'vod' | 'live'
|
||||
}) {
|
||||
const { availableEncoders, profile, streamType, videoType } = options
|
||||
|
||||
const encodersToTry = availableEncoders.encodersToTry[videoType][streamType]
|
||||
const encoders = availableEncoders.available[videoType]
|
||||
|
||||
for (const encoder of encodersToTry) {
|
||||
if (!(await checkFFmpegEncoders(availableEncoders)).get(encoder)) {
|
||||
logger.debug('Encoder %s not available in ffmpeg, skipping.', encoder, lTags())
|
||||
continue
|
||||
}
|
||||
|
||||
if (!encoders[encoder]) {
|
||||
logger.debug('Encoder %s not available in peertube encoders, skipping.', encoder, lTags())
|
||||
continue
|
||||
}
|
||||
|
||||
// An object containing available profiles for this encoder
|
||||
const builderProfiles: EncoderProfile<EncoderOptionsBuilder> = encoders[encoder]
|
||||
let builder = builderProfiles[profile]
|
||||
|
||||
if (!builder) {
|
||||
logger.debug('Profile %s for encoder %s not available. Fallback to default.', profile, encoder, lTags())
|
||||
builder = builderProfiles.default
|
||||
|
||||
if (!builder) {
|
||||
logger.debug('Default profile for encoder %s not available. Try next available encoder.', encoder, lTags())
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
const result = await builder(
|
||||
pick(options, [
|
||||
'input',
|
||||
'canCopyAudio',
|
||||
'canCopyVideo',
|
||||
'resolution',
|
||||
'inputBitrate',
|
||||
'fps',
|
||||
'inputRatio',
|
||||
'streamNum'
|
||||
])
|
||||
)
|
||||
|
||||
return {
|
||||
result,
|
||||
|
||||
// If we don't have output options, then copy the input stream
|
||||
encoder: result.copy === true
|
||||
? 'copy'
|
||||
: encoder
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
export {
|
||||
checkFFmpegEncoders,
|
||||
resetSupportedEncoders,
|
||||
|
||||
getEncoderBuilderResult
|
||||
}
|
14
server/helpers/ffmpeg/ffmpeg-image.ts
Normal file
14
server/helpers/ffmpeg/ffmpeg-image.ts
Normal file
@ -0,0 +1,14 @@
|
||||
import { FFmpegImage } from '@shared/ffmpeg'
|
||||
import { getFFmpegCommandWrapperOptions } from './ffmpeg-options'
|
||||
|
||||
export function processGIF (options: Parameters<FFmpegImage['processGIF']>[0]) {
|
||||
return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).processGIF(options)
|
||||
}
|
||||
|
||||
export function generateThumbnailFromVideo (options: Parameters<FFmpegImage['generateThumbnailFromVideo']>[0]) {
|
||||
return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).generateThumbnailFromVideo(options)
|
||||
}
|
||||
|
||||
export function convertWebPToJPG (options: Parameters<FFmpegImage['convertWebPToJPG']>[0]) {
|
||||
return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).convertWebPToJPG(options)
|
||||
}
|
@ -1,46 +0,0 @@
|
||||
import ffmpeg from 'fluent-ffmpeg'
|
||||
import { FFMPEG_NICE } from '@server/initializers/constants'
|
||||
import { runCommand } from './ffmpeg-commons'
|
||||
|
||||
function convertWebPToJPG (path: string, destination: string): Promise<void> {
|
||||
const command = ffmpeg(path, { niceness: FFMPEG_NICE.THUMBNAIL })
|
||||
.output(destination)
|
||||
|
||||
return runCommand({ command, silent: true })
|
||||
}
|
||||
|
||||
function processGIF (
|
||||
path: string,
|
||||
destination: string,
|
||||
newSize: { width: number, height: number }
|
||||
): Promise<void> {
|
||||
const command = ffmpeg(path, { niceness: FFMPEG_NICE.THUMBNAIL })
|
||||
.fps(20)
|
||||
.size(`${newSize.width}x${newSize.height}`)
|
||||
.output(destination)
|
||||
|
||||
return runCommand({ command })
|
||||
}
|
||||
|
||||
async function generateThumbnailFromVideo (fromPath: string, folder: string, imageName: string) {
|
||||
const pendingImageName = 'pending-' + imageName
|
||||
|
||||
const options = {
|
||||
filename: pendingImageName,
|
||||
count: 1,
|
||||
folder
|
||||
}
|
||||
|
||||
return new Promise<string>((res, rej) => {
|
||||
ffmpeg(fromPath, { niceness: FFMPEG_NICE.THUMBNAIL })
|
||||
.on('error', rej)
|
||||
.on('end', () => res(imageName))
|
||||
.thumbnail(options)
|
||||
})
|
||||
}
|
||||
|
||||
export {
|
||||
convertWebPToJPG,
|
||||
processGIF,
|
||||
generateThumbnailFromVideo
|
||||
}
|
@ -1,204 +0,0 @@
|
||||
import { FfmpegCommand, FilterSpecification } from 'fluent-ffmpeg'
|
||||
import { join } from 'path'
|
||||
import { VIDEO_LIVE } from '@server/initializers/constants'
|
||||
import { AvailableEncoders, LiveVideoLatencyMode } from '@shared/models'
|
||||
import { logger, loggerTagsFactory } from '../logger'
|
||||
import { buildStreamSuffix, getFFmpeg, getScaleFilter, StreamType } from './ffmpeg-commons'
|
||||
import { getEncoderBuilderResult } from './ffmpeg-encoders'
|
||||
import { addDefaultEncoderGlobalParams, addDefaultEncoderParams, applyEncoderOptions } from './ffmpeg-presets'
|
||||
import { computeFPS } from './ffprobe-utils'
|
||||
|
||||
const lTags = loggerTagsFactory('ffmpeg')
|
||||
|
||||
async function getLiveTranscodingCommand (options: {
|
||||
inputUrl: string
|
||||
|
||||
outPath: string
|
||||
masterPlaylistName: string
|
||||
latencyMode: LiveVideoLatencyMode
|
||||
|
||||
resolutions: number[]
|
||||
|
||||
// Input information
|
||||
fps: number
|
||||
bitrate: number
|
||||
ratio: number
|
||||
hasAudio: boolean
|
||||
|
||||
availableEncoders: AvailableEncoders
|
||||
profile: string
|
||||
}) {
|
||||
const {
|
||||
inputUrl,
|
||||
outPath,
|
||||
resolutions,
|
||||
fps,
|
||||
bitrate,
|
||||
availableEncoders,
|
||||
profile,
|
||||
masterPlaylistName,
|
||||
ratio,
|
||||
latencyMode,
|
||||
hasAudio
|
||||
} = options
|
||||
|
||||
const command = getFFmpeg(inputUrl, 'live')
|
||||
|
||||
const varStreamMap: string[] = []
|
||||
|
||||
const complexFilter: FilterSpecification[] = [
|
||||
{
|
||||
inputs: '[v:0]',
|
||||
filter: 'split',
|
||||
options: resolutions.length,
|
||||
outputs: resolutions.map(r => `vtemp${r}`)
|
||||
}
|
||||
]
|
||||
|
||||
command.outputOption('-sc_threshold 0')
|
||||
|
||||
addDefaultEncoderGlobalParams(command)
|
||||
|
||||
for (let i = 0; i < resolutions.length; i++) {
|
||||
const streamMap: string[] = []
|
||||
const resolution = resolutions[i]
|
||||
const resolutionFPS = computeFPS(fps, resolution)
|
||||
|
||||
const baseEncoderBuilderParams = {
|
||||
input: inputUrl,
|
||||
|
||||
availableEncoders,
|
||||
profile,
|
||||
|
||||
canCopyAudio: true,
|
||||
canCopyVideo: true,
|
||||
|
||||
inputBitrate: bitrate,
|
||||
inputRatio: ratio,
|
||||
|
||||
resolution,
|
||||
fps: resolutionFPS,
|
||||
|
||||
streamNum: i,
|
||||
videoType: 'live' as 'live'
|
||||
}
|
||||
|
||||
{
|
||||
const streamType: StreamType = 'video'
|
||||
const builderResult = await getEncoderBuilderResult({ ...baseEncoderBuilderParams, streamType })
|
||||
if (!builderResult) {
|
||||
throw new Error('No available live video encoder found')
|
||||
}
|
||||
|
||||
command.outputOption(`-map [vout${resolution}]`)
|
||||
|
||||
addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps: resolutionFPS, streamNum: i })
|
||||
|
||||
logger.debug(
|
||||
'Apply ffmpeg live video params from %s using %s profile.', builderResult.encoder, profile,
|
||||
{ builderResult, fps: resolutionFPS, resolution, ...lTags() }
|
||||
)
|
||||
|
||||
command.outputOption(`${buildStreamSuffix('-c:v', i)} ${builderResult.encoder}`)
|
||||
applyEncoderOptions(command, builderResult.result)
|
||||
|
||||
complexFilter.push({
|
||||
inputs: `vtemp${resolution}`,
|
||||
filter: getScaleFilter(builderResult.result),
|
||||
options: `w=-2:h=${resolution}`,
|
||||
outputs: `vout${resolution}`
|
||||
})
|
||||
|
||||
streamMap.push(`v:${i}`)
|
||||
}
|
||||
|
||||
if (hasAudio) {
|
||||
const streamType: StreamType = 'audio'
|
||||
const builderResult = await getEncoderBuilderResult({ ...baseEncoderBuilderParams, streamType })
|
||||
if (!builderResult) {
|
||||
throw new Error('No available live audio encoder found')
|
||||
}
|
||||
|
||||
command.outputOption('-map a:0')
|
||||
|
||||
addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps: resolutionFPS, streamNum: i })
|
||||
|
||||
logger.debug(
|
||||
'Apply ffmpeg live audio params from %s using %s profile.', builderResult.encoder, profile,
|
||||
{ builderResult, fps: resolutionFPS, resolution, ...lTags() }
|
||||
)
|
||||
|
||||
command.outputOption(`${buildStreamSuffix('-c:a', i)} ${builderResult.encoder}`)
|
||||
applyEncoderOptions(command, builderResult.result)
|
||||
|
||||
streamMap.push(`a:${i}`)
|
||||
}
|
||||
|
||||
varStreamMap.push(streamMap.join(','))
|
||||
}
|
||||
|
||||
command.complexFilter(complexFilter)
|
||||
|
||||
addDefaultLiveHLSParams({ command, outPath, masterPlaylistName, latencyMode })
|
||||
|
||||
command.outputOption('-var_stream_map', varStreamMap.join(' '))
|
||||
|
||||
return command
|
||||
}
|
||||
|
||||
function getLiveMuxingCommand (options: {
|
||||
inputUrl: string
|
||||
outPath: string
|
||||
masterPlaylistName: string
|
||||
latencyMode: LiveVideoLatencyMode
|
||||
}) {
|
||||
const { inputUrl, outPath, masterPlaylistName, latencyMode } = options
|
||||
|
||||
const command = getFFmpeg(inputUrl, 'live')
|
||||
|
||||
command.outputOption('-c:v copy')
|
||||
command.outputOption('-c:a copy')
|
||||
command.outputOption('-map 0:a?')
|
||||
command.outputOption('-map 0:v?')
|
||||
|
||||
addDefaultLiveHLSParams({ command, outPath, masterPlaylistName, latencyMode })
|
||||
|
||||
return command
|
||||
}
|
||||
|
||||
function getLiveSegmentTime (latencyMode: LiveVideoLatencyMode) {
|
||||
if (latencyMode === LiveVideoLatencyMode.SMALL_LATENCY) {
|
||||
return VIDEO_LIVE.SEGMENT_TIME_SECONDS.SMALL_LATENCY
|
||||
}
|
||||
|
||||
return VIDEO_LIVE.SEGMENT_TIME_SECONDS.DEFAULT_LATENCY
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
getLiveSegmentTime,
|
||||
|
||||
getLiveTranscodingCommand,
|
||||
getLiveMuxingCommand
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function addDefaultLiveHLSParams (options: {
|
||||
command: FfmpegCommand
|
||||
outPath: string
|
||||
masterPlaylistName: string
|
||||
latencyMode: LiveVideoLatencyMode
|
||||
}) {
|
||||
const { command, outPath, masterPlaylistName, latencyMode } = options
|
||||
|
||||
command.outputOption('-hls_time ' + getLiveSegmentTime(latencyMode))
|
||||
command.outputOption('-hls_list_size ' + VIDEO_LIVE.SEGMENTS_LIST_SIZE)
|
||||
command.outputOption('-hls_flags delete_segments+independent_segments+program_date_time')
|
||||
command.outputOption(`-hls_segment_filename ${join(outPath, '%v-%06d.ts')}`)
|
||||
command.outputOption('-master_pl_name ' + masterPlaylistName)
|
||||
command.outputOption(`-f hls`)
|
||||
|
||||
command.output(join(outPath, '%v.m3u8'))
|
||||
}
|
45
server/helpers/ffmpeg/ffmpeg-options.ts
Normal file
45
server/helpers/ffmpeg/ffmpeg-options.ts
Normal file
@ -0,0 +1,45 @@
|
||||
import { logger } from '@server/helpers/logger'
|
||||
import { CONFIG } from '@server/initializers/config'
|
||||
import { FFMPEG_NICE } from '@server/initializers/constants'
|
||||
import { FFmpegCommandWrapperOptions } from '@shared/ffmpeg'
|
||||
import { AvailableEncoders } from '@shared/models'
|
||||
|
||||
type CommandType = 'live' | 'vod' | 'thumbnail'
|
||||
|
||||
export function getFFmpegCommandWrapperOptions (type: CommandType, availableEncoders?: AvailableEncoders): FFmpegCommandWrapperOptions {
|
||||
return {
|
||||
availableEncoders,
|
||||
profile: getProfile(type),
|
||||
|
||||
niceness: FFMPEG_NICE[type],
|
||||
tmpDirectory: CONFIG.STORAGE.TMP_DIR,
|
||||
threads: getThreads(type),
|
||||
|
||||
logger: {
|
||||
debug: logger.debug.bind(logger),
|
||||
info: logger.info.bind(logger),
|
||||
warn: logger.warn.bind(logger),
|
||||
error: logger.error.bind(logger)
|
||||
},
|
||||
lTags: { tags: [ 'ffmpeg' ] }
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Private
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function getThreads (type: CommandType) {
|
||||
if (type === 'live') return CONFIG.LIVE.TRANSCODING.THREADS
|
||||
if (type === 'vod') return CONFIG.TRANSCODING.THREADS
|
||||
|
||||
// Auto
|
||||
return 0
|
||||
}
|
||||
|
||||
function getProfile (type: CommandType) {
|
||||
if (type === 'live') return CONFIG.LIVE.TRANSCODING.PROFILE
|
||||
if (type === 'vod') return CONFIG.TRANSCODING.PROFILE
|
||||
|
||||
return undefined
|
||||
}
|
@ -1,156 +0,0 @@
|
||||
import { FfmpegCommand } from 'fluent-ffmpeg'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger'
|
||||
import { pick } from '@shared/core-utils'
|
||||
import { AvailableEncoders, EncoderOptions } from '@shared/models'
|
||||
import { buildStreamSuffix, getScaleFilter, StreamType } from './ffmpeg-commons'
|
||||
import { getEncoderBuilderResult } from './ffmpeg-encoders'
|
||||
import { ffprobePromise, getVideoStreamBitrate, getVideoStreamDimensionsInfo, hasAudioStream } from './ffprobe-utils'
|
||||
|
||||
const lTags = loggerTagsFactory('ffmpeg')
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function addDefaultEncoderGlobalParams (command: FfmpegCommand) {
|
||||
// avoid issues when transcoding some files: https://trac.ffmpeg.org/ticket/6375
|
||||
command.outputOption('-max_muxing_queue_size 1024')
|
||||
// strip all metadata
|
||||
.outputOption('-map_metadata -1')
|
||||
// allows import of source material with incompatible pixel formats (e.g. MJPEG video)
|
||||
.outputOption('-pix_fmt yuv420p')
|
||||
}
|
||||
|
||||
function addDefaultEncoderParams (options: {
|
||||
command: FfmpegCommand
|
||||
encoder: 'libx264' | string
|
||||
fps: number
|
||||
|
||||
streamNum?: number
|
||||
}) {
|
||||
const { command, encoder, fps, streamNum } = options
|
||||
|
||||
if (encoder === 'libx264') {
|
||||
// 3.1 is the minimal resource allocation for our highest supported resolution
|
||||
command.outputOption(buildStreamSuffix('-level:v', streamNum) + ' 3.1')
|
||||
|
||||
if (fps) {
|
||||
// Keyframe interval of 2 seconds for faster seeking and resolution switching.
|
||||
// https://streaminglearningcenter.com/blogs/whats-the-right-keyframe-interval.html
|
||||
// https://superuser.com/a/908325
|
||||
command.outputOption(buildStreamSuffix('-g:v', streamNum) + ' ' + (fps * 2))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function presetVOD (options: {
|
||||
command: FfmpegCommand
|
||||
input: string
|
||||
|
||||
availableEncoders: AvailableEncoders
|
||||
profile: string
|
||||
|
||||
canCopyAudio: boolean
|
||||
canCopyVideo: boolean
|
||||
|
||||
resolution: number
|
||||
fps: number
|
||||
|
||||
scaleFilterValue?: string
|
||||
}) {
|
||||
const { command, input, profile, resolution, fps, scaleFilterValue } = options
|
||||
|
||||
let localCommand = command
|
||||
.format('mp4')
|
||||
.outputOption('-movflags faststart')
|
||||
|
||||
addDefaultEncoderGlobalParams(command)
|
||||
|
||||
const probe = await ffprobePromise(input)
|
||||
|
||||
// Audio encoder
|
||||
const bitrate = await getVideoStreamBitrate(input, probe)
|
||||
const videoStreamDimensions = await getVideoStreamDimensionsInfo(input, probe)
|
||||
|
||||
let streamsToProcess: StreamType[] = [ 'audio', 'video' ]
|
||||
|
||||
if (!await hasAudioStream(input, probe)) {
|
||||
localCommand = localCommand.noAudio()
|
||||
streamsToProcess = [ 'video' ]
|
||||
}
|
||||
|
||||
for (const streamType of streamsToProcess) {
|
||||
const builderResult = await getEncoderBuilderResult({
|
||||
...pick(options, [ 'availableEncoders', 'canCopyAudio', 'canCopyVideo' ]),
|
||||
|
||||
input,
|
||||
inputBitrate: bitrate,
|
||||
inputRatio: videoStreamDimensions?.ratio || 0,
|
||||
|
||||
profile,
|
||||
resolution,
|
||||
fps,
|
||||
streamType,
|
||||
|
||||
videoType: 'vod' as 'vod'
|
||||
})
|
||||
|
||||
if (!builderResult) {
|
||||
throw new Error('No available encoder found for stream ' + streamType)
|
||||
}
|
||||
|
||||
logger.debug(
|
||||
'Apply ffmpeg params from %s for %s stream of input %s using %s profile.',
|
||||
builderResult.encoder, streamType, input, profile,
|
||||
{ builderResult, resolution, fps, ...lTags() }
|
||||
)
|
||||
|
||||
if (streamType === 'video') {
|
||||
localCommand.videoCodec(builderResult.encoder)
|
||||
|
||||
if (scaleFilterValue) {
|
||||
localCommand.outputOption(`-vf ${getScaleFilter(builderResult.result)}=${scaleFilterValue}`)
|
||||
}
|
||||
} else if (streamType === 'audio') {
|
||||
localCommand.audioCodec(builderResult.encoder)
|
||||
}
|
||||
|
||||
applyEncoderOptions(localCommand, builderResult.result)
|
||||
addDefaultEncoderParams({ command: localCommand, encoder: builderResult.encoder, fps })
|
||||
}
|
||||
|
||||
return localCommand
|
||||
}
|
||||
|
||||
function presetCopy (command: FfmpegCommand): FfmpegCommand {
|
||||
return command
|
||||
.format('mp4')
|
||||
.videoCodec('copy')
|
||||
.audioCodec('copy')
|
||||
}
|
||||
|
||||
function presetOnlyAudio (command: FfmpegCommand): FfmpegCommand {
|
||||
return command
|
||||
.format('mp4')
|
||||
.audioCodec('copy')
|
||||
.noVideo()
|
||||
}
|
||||
|
||||
function applyEncoderOptions (command: FfmpegCommand, options: EncoderOptions): FfmpegCommand {
|
||||
return command
|
||||
.inputOptions(options.inputOptions ?? [])
|
||||
.outputOptions(options.outputOptions ?? [])
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
presetVOD,
|
||||
presetCopy,
|
||||
presetOnlyAudio,
|
||||
|
||||
addDefaultEncoderGlobalParams,
|
||||
addDefaultEncoderParams,
|
||||
|
||||
applyEncoderOptions
|
||||
}
|
@ -1,267 +0,0 @@
|
||||
import { MutexInterface } from 'async-mutex'
|
||||
import { Job } from 'bullmq'
|
||||
import { FfmpegCommand } from 'fluent-ffmpeg'
|
||||
import { readFile, writeFile } from 'fs-extra'
|
||||
import { dirname } from 'path'
|
||||
import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
|
||||
import { pick } from '@shared/core-utils'
|
||||
import { AvailableEncoders, VideoResolution } from '@shared/models'
|
||||
import { logger, loggerTagsFactory } from '../logger'
|
||||
import { getFFmpeg, runCommand } from './ffmpeg-commons'
|
||||
import { presetCopy, presetOnlyAudio, presetVOD } from './ffmpeg-presets'
|
||||
import { computeFPS, ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS } from './ffprobe-utils'
|
||||
|
||||
const lTags = loggerTagsFactory('ffmpeg')
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type TranscodeVODOptionsType = 'hls' | 'hls-from-ts' | 'quick-transcode' | 'video' | 'merge-audio' | 'only-audio'
|
||||
|
||||
interface BaseTranscodeVODOptions {
|
||||
type: TranscodeVODOptionsType
|
||||
|
||||
inputPath: string
|
||||
outputPath: string
|
||||
|
||||
// Will be released after the ffmpeg started
|
||||
// To prevent a bug where the input file does not exist anymore when running ffmpeg
|
||||
inputFileMutexReleaser: MutexInterface.Releaser
|
||||
|
||||
availableEncoders: AvailableEncoders
|
||||
profile: string
|
||||
|
||||
resolution: number
|
||||
|
||||
job?: Job
|
||||
}
|
||||
|
||||
interface HLSTranscodeOptions extends BaseTranscodeVODOptions {
|
||||
type: 'hls'
|
||||
copyCodecs: boolean
|
||||
hlsPlaylist: {
|
||||
videoFilename: string
|
||||
}
|
||||
}
|
||||
|
||||
interface HLSFromTSTranscodeOptions extends BaseTranscodeVODOptions {
|
||||
type: 'hls-from-ts'
|
||||
|
||||
isAAC: boolean
|
||||
|
||||
hlsPlaylist: {
|
||||
videoFilename: string
|
||||
}
|
||||
}
|
||||
|
||||
interface QuickTranscodeOptions extends BaseTranscodeVODOptions {
|
||||
type: 'quick-transcode'
|
||||
}
|
||||
|
||||
interface VideoTranscodeOptions extends BaseTranscodeVODOptions {
|
||||
type: 'video'
|
||||
}
|
||||
|
||||
interface MergeAudioTranscodeOptions extends BaseTranscodeVODOptions {
|
||||
type: 'merge-audio'
|
||||
audioPath: string
|
||||
}
|
||||
|
||||
interface OnlyAudioTranscodeOptions extends BaseTranscodeVODOptions {
|
||||
type: 'only-audio'
|
||||
}
|
||||
|
||||
type TranscodeVODOptions =
|
||||
HLSTranscodeOptions
|
||||
| HLSFromTSTranscodeOptions
|
||||
| VideoTranscodeOptions
|
||||
| MergeAudioTranscodeOptions
|
||||
| OnlyAudioTranscodeOptions
|
||||
| QuickTranscodeOptions
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const builders: {
|
||||
[ type in TranscodeVODOptionsType ]: (c: FfmpegCommand, o?: TranscodeVODOptions) => Promise<FfmpegCommand> | FfmpegCommand
|
||||
} = {
|
||||
'quick-transcode': buildQuickTranscodeCommand,
|
||||
'hls': buildHLSVODCommand,
|
||||
'hls-from-ts': buildHLSVODFromTSCommand,
|
||||
'merge-audio': buildAudioMergeCommand,
|
||||
'only-audio': buildOnlyAudioCommand,
|
||||
'video': buildVODCommand
|
||||
}
|
||||
|
||||
async function transcodeVOD (options: TranscodeVODOptions) {
|
||||
logger.debug('Will run transcode.', { options, ...lTags() })
|
||||
|
||||
let command = getFFmpeg(options.inputPath, 'vod')
|
||||
.output(options.outputPath)
|
||||
|
||||
command = await builders[options.type](command, options)
|
||||
|
||||
command.on('start', () => {
|
||||
setTimeout(() => {
|
||||
options.inputFileMutexReleaser()
|
||||
}, 1000)
|
||||
})
|
||||
|
||||
await runCommand({ command, job: options.job })
|
||||
|
||||
await fixHLSPlaylistIfNeeded(options)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
transcodeVOD,
|
||||
|
||||
buildVODCommand,
|
||||
|
||||
TranscodeVODOptions,
|
||||
TranscodeVODOptionsType
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function buildVODCommand (command: FfmpegCommand, options: TranscodeVODOptions) {
|
||||
const probe = await ffprobePromise(options.inputPath)
|
||||
|
||||
let fps = await getVideoStreamFPS(options.inputPath, probe)
|
||||
fps = computeFPS(fps, options.resolution)
|
||||
|
||||
let scaleFilterValue: string
|
||||
|
||||
if (options.resolution !== undefined) {
|
||||
const videoStreamInfo = await getVideoStreamDimensionsInfo(options.inputPath, probe)
|
||||
|
||||
scaleFilterValue = videoStreamInfo?.isPortraitMode === true
|
||||
? `w=${options.resolution}:h=-2`
|
||||
: `w=-2:h=${options.resolution}`
|
||||
}
|
||||
|
||||
command = await presetVOD({
|
||||
...pick(options, [ 'resolution', 'availableEncoders', 'profile' ]),
|
||||
|
||||
command,
|
||||
input: options.inputPath,
|
||||
canCopyAudio: true,
|
||||
canCopyVideo: true,
|
||||
fps,
|
||||
scaleFilterValue
|
||||
})
|
||||
|
||||
return command
|
||||
}
|
||||
|
||||
function buildQuickTranscodeCommand (command: FfmpegCommand) {
|
||||
command = presetCopy(command)
|
||||
|
||||
command = command.outputOption('-map_metadata -1') // strip all metadata
|
||||
.outputOption('-movflags faststart')
|
||||
|
||||
return command
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Audio transcoding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function buildAudioMergeCommand (command: FfmpegCommand, options: MergeAudioTranscodeOptions) {
|
||||
command = command.loop(undefined)
|
||||
|
||||
const scaleFilterValue = getMergeAudioScaleFilterValue()
|
||||
command = await presetVOD({
|
||||
...pick(options, [ 'resolution', 'availableEncoders', 'profile' ]),
|
||||
|
||||
command,
|
||||
input: options.audioPath,
|
||||
canCopyAudio: true,
|
||||
canCopyVideo: true,
|
||||
fps: VIDEO_TRANSCODING_FPS.AUDIO_MERGE,
|
||||
scaleFilterValue
|
||||
})
|
||||
|
||||
command.outputOption('-preset:v veryfast')
|
||||
|
||||
command = command.input(options.audioPath)
|
||||
.outputOption('-tune stillimage')
|
||||
.outputOption('-shortest')
|
||||
|
||||
return command
|
||||
}
|
||||
|
||||
function buildOnlyAudioCommand (command: FfmpegCommand, _options: OnlyAudioTranscodeOptions) {
|
||||
command = presetOnlyAudio(command)
|
||||
|
||||
return command
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// HLS transcoding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function buildHLSVODCommand (command: FfmpegCommand, options: HLSTranscodeOptions) {
|
||||
const videoPath = getHLSVideoPath(options)
|
||||
|
||||
if (options.copyCodecs) command = presetCopy(command)
|
||||
else if (options.resolution === VideoResolution.H_NOVIDEO) command = presetOnlyAudio(command)
|
||||
else command = await buildVODCommand(command, options)
|
||||
|
||||
addCommonHLSVODCommandOptions(command, videoPath)
|
||||
|
||||
return command
|
||||
}
|
||||
|
||||
function buildHLSVODFromTSCommand (command: FfmpegCommand, options: HLSFromTSTranscodeOptions) {
|
||||
const videoPath = getHLSVideoPath(options)
|
||||
|
||||
command.outputOption('-c copy')
|
||||
|
||||
if (options.isAAC) {
|
||||
// Required for example when copying an AAC stream from an MPEG-TS
|
||||
// Since it's a bitstream filter, we don't need to reencode the audio
|
||||
command.outputOption('-bsf:a aac_adtstoasc')
|
||||
}
|
||||
|
||||
addCommonHLSVODCommandOptions(command, videoPath)
|
||||
|
||||
return command
|
||||
}
|
||||
|
||||
function addCommonHLSVODCommandOptions (command: FfmpegCommand, outputPath: string) {
|
||||
return command.outputOption('-hls_time 4')
|
||||
.outputOption('-hls_list_size 0')
|
||||
.outputOption('-hls_playlist_type vod')
|
||||
.outputOption('-hls_segment_filename ' + outputPath)
|
||||
.outputOption('-hls_segment_type fmp4')
|
||||
.outputOption('-f hls')
|
||||
.outputOption('-hls_flags single_file')
|
||||
}
|
||||
|
||||
async function fixHLSPlaylistIfNeeded (options: TranscodeVODOptions) {
|
||||
if (options.type !== 'hls' && options.type !== 'hls-from-ts') return
|
||||
|
||||
const fileContent = await readFile(options.outputPath)
|
||||
|
||||
const videoFileName = options.hlsPlaylist.videoFilename
|
||||
const videoFilePath = getHLSVideoPath(options)
|
||||
|
||||
// Fix wrong mapping with some ffmpeg versions
|
||||
const newContent = fileContent.toString()
|
||||
.replace(`#EXT-X-MAP:URI="${videoFilePath}",`, `#EXT-X-MAP:URI="${videoFileName}",`)
|
||||
|
||||
await writeFile(options.outputPath, newContent)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function getHLSVideoPath (options: HLSTranscodeOptions | HLSFromTSTranscodeOptions) {
|
||||
return `${dirname(options.outputPath)}/${options.hlsPlaylist.videoFilename}`
|
||||
}
|
||||
|
||||
// Avoid "height not divisible by 2" error
|
||||
function getMergeAudioScaleFilterValue () {
|
||||
return 'trunc(iw/2)*2:trunc(ih/2)*2'
|
||||
}
|
@ -1,254 +0,0 @@
|
||||
import { FfprobeData } from 'fluent-ffmpeg'
|
||||
import { getMaxBitrate } from '@shared/core-utils'
|
||||
import {
|
||||
buildFileMetadata,
|
||||
ffprobePromise,
|
||||
getAudioStream,
|
||||
getMaxAudioBitrate,
|
||||
getVideoStream,
|
||||
getVideoStreamBitrate,
|
||||
getVideoStreamDimensionsInfo,
|
||||
getVideoStreamDuration,
|
||||
getVideoStreamFPS,
|
||||
hasAudioStream
|
||||
} from '@shared/extra-utils/ffprobe'
|
||||
import { VideoResolution, VideoTranscodingFPS } from '@shared/models'
|
||||
import { CONFIG } from '../../initializers/config'
|
||||
import { VIDEO_TRANSCODING_FPS } from '../../initializers/constants'
|
||||
import { toEven } from '../core-utils'
|
||||
import { logger } from '../logger'
|
||||
|
||||
/**
|
||||
*
|
||||
* Helpers to run ffprobe and extract data from the JSON output
|
||||
*
|
||||
*/
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Codecs
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function getVideoStreamCodec (path: string) {
|
||||
const videoStream = await getVideoStream(path)
|
||||
if (!videoStream) return ''
|
||||
|
||||
const videoCodec = videoStream.codec_tag_string
|
||||
|
||||
if (videoCodec === 'vp09') return 'vp09.00.50.08'
|
||||
if (videoCodec === 'hev1') return 'hev1.1.6.L93.B0'
|
||||
|
||||
const baseProfileMatrix = {
|
||||
avc1: {
|
||||
High: '6400',
|
||||
Main: '4D40',
|
||||
Baseline: '42E0'
|
||||
},
|
||||
av01: {
|
||||
High: '1',
|
||||
Main: '0',
|
||||
Professional: '2'
|
||||
}
|
||||
}
|
||||
|
||||
let baseProfile = baseProfileMatrix[videoCodec][videoStream.profile]
|
||||
if (!baseProfile) {
|
||||
logger.warn('Cannot get video profile codec of %s.', path, { videoStream })
|
||||
baseProfile = baseProfileMatrix[videoCodec]['High'] // Fallback
|
||||
}
|
||||
|
||||
if (videoCodec === 'av01') {
|
||||
let level = videoStream.level.toString()
|
||||
if (level.length === 1) level = `0${level}`
|
||||
|
||||
// Guess the tier indicator and bit depth
|
||||
return `${videoCodec}.${baseProfile}.${level}M.08`
|
||||
}
|
||||
|
||||
let level = videoStream.level.toString(16)
|
||||
if (level.length === 1) level = `0${level}`
|
||||
|
||||
// Default, h264 codec
|
||||
return `${videoCodec}.${baseProfile}${level}`
|
||||
}
|
||||
|
||||
async function getAudioStreamCodec (path: string, existingProbe?: FfprobeData) {
|
||||
const { audioStream } = await getAudioStream(path, existingProbe)
|
||||
|
||||
if (!audioStream) return ''
|
||||
|
||||
const audioCodecName = audioStream.codec_name
|
||||
|
||||
if (audioCodecName === 'opus') return 'opus'
|
||||
if (audioCodecName === 'vorbis') return 'vorbis'
|
||||
if (audioCodecName === 'aac') return 'mp4a.40.2'
|
||||
if (audioCodecName === 'mp3') return 'mp4a.40.34'
|
||||
|
||||
logger.warn('Cannot get audio codec of %s.', path, { audioStream })
|
||||
|
||||
return 'mp4a.40.2' // Fallback
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Resolutions
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function computeResolutionsToTranscode (options: {
|
||||
input: number
|
||||
type: 'vod' | 'live'
|
||||
includeInput: boolean
|
||||
strictLower: boolean
|
||||
hasAudio: boolean
|
||||
}) {
|
||||
const { input, type, includeInput, strictLower, hasAudio } = options
|
||||
|
||||
const configResolutions = type === 'vod'
|
||||
? CONFIG.TRANSCODING.RESOLUTIONS
|
||||
: CONFIG.LIVE.TRANSCODING.RESOLUTIONS
|
||||
|
||||
const resolutionsEnabled = new Set<number>()
|
||||
|
||||
// Put in the order we want to proceed jobs
|
||||
const availableResolutions: VideoResolution[] = [
|
||||
VideoResolution.H_NOVIDEO,
|
||||
VideoResolution.H_480P,
|
||||
VideoResolution.H_360P,
|
||||
VideoResolution.H_720P,
|
||||
VideoResolution.H_240P,
|
||||
VideoResolution.H_144P,
|
||||
VideoResolution.H_1080P,
|
||||
VideoResolution.H_1440P,
|
||||
VideoResolution.H_4K
|
||||
]
|
||||
|
||||
for (const resolution of availableResolutions) {
|
||||
// Resolution not enabled
|
||||
if (configResolutions[resolution + 'p'] !== true) continue
|
||||
// Too big resolution for input file
|
||||
if (input < resolution) continue
|
||||
// We only want lower resolutions than input file
|
||||
if (strictLower && input === resolution) continue
|
||||
// Audio resolutio but no audio in the video
|
||||
if (resolution === VideoResolution.H_NOVIDEO && !hasAudio) continue
|
||||
|
||||
resolutionsEnabled.add(resolution)
|
||||
}
|
||||
|
||||
if (includeInput) {
|
||||
// Always use an even resolution to avoid issues with ffmpeg
|
||||
resolutionsEnabled.add(toEven(input))
|
||||
}
|
||||
|
||||
return Array.from(resolutionsEnabled)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Can quick transcode
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function canDoQuickTranscode (path: string): Promise<boolean> {
|
||||
if (CONFIG.TRANSCODING.PROFILE !== 'default') return false
|
||||
|
||||
const probe = await ffprobePromise(path)
|
||||
|
||||
return await canDoQuickVideoTranscode(path, probe) &&
|
||||
await canDoQuickAudioTranscode(path, probe)
|
||||
}
|
||||
|
||||
async function canDoQuickAudioTranscode (path: string, probe?: FfprobeData): Promise<boolean> {
|
||||
const parsedAudio = await getAudioStream(path, probe)
|
||||
|
||||
if (!parsedAudio.audioStream) return true
|
||||
|
||||
if (parsedAudio.audioStream['codec_name'] !== 'aac') return false
|
||||
|
||||
const audioBitrate = parsedAudio.bitrate
|
||||
if (!audioBitrate) return false
|
||||
|
||||
const maxAudioBitrate = getMaxAudioBitrate('aac', audioBitrate)
|
||||
if (maxAudioBitrate !== -1 && audioBitrate > maxAudioBitrate) return false
|
||||
|
||||
const channelLayout = parsedAudio.audioStream['channel_layout']
|
||||
// Causes playback issues with Chrome
|
||||
if (!channelLayout || channelLayout === 'unknown' || channelLayout === 'quad') return false
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
async function canDoQuickVideoTranscode (path: string, probe?: FfprobeData): Promise<boolean> {
|
||||
const videoStream = await getVideoStream(path, probe)
|
||||
const fps = await getVideoStreamFPS(path, probe)
|
||||
const bitRate = await getVideoStreamBitrate(path, probe)
|
||||
const resolutionData = await getVideoStreamDimensionsInfo(path, probe)
|
||||
|
||||
// If ffprobe did not manage to guess the bitrate
|
||||
if (!bitRate) return false
|
||||
|
||||
// check video params
|
||||
if (!videoStream) return false
|
||||
if (videoStream['codec_name'] !== 'h264') return false
|
||||
if (videoStream['pix_fmt'] !== 'yuv420p') return false
|
||||
if (fps < VIDEO_TRANSCODING_FPS.MIN || fps > VIDEO_TRANSCODING_FPS.MAX) return false
|
||||
if (bitRate > getMaxBitrate({ ...resolutionData, fps })) return false
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Framerate
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function getClosestFramerateStandard <K extends keyof Pick<VideoTranscodingFPS, 'HD_STANDARD' | 'STANDARD'>> (fps: number, type: K) {
|
||||
return VIDEO_TRANSCODING_FPS[type].slice(0)
|
||||
.sort((a, b) => fps % a - fps % b)[0]
|
||||
}
|
||||
|
||||
function computeFPS (fpsArg: number, resolution: VideoResolution) {
|
||||
let fps = fpsArg
|
||||
|
||||
if (
|
||||
// On small/medium resolutions, limit FPS
|
||||
resolution !== undefined &&
|
||||
resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN &&
|
||||
fps > VIDEO_TRANSCODING_FPS.AVERAGE
|
||||
) {
|
||||
// Get closest standard framerate by modulo: downsampling has to be done to a divisor of the nominal fps value
|
||||
fps = getClosestFramerateStandard(fps, 'STANDARD')
|
||||
}
|
||||
|
||||
// Hard FPS limits
|
||||
if (fps > VIDEO_TRANSCODING_FPS.MAX) fps = getClosestFramerateStandard(fps, 'HD_STANDARD')
|
||||
|
||||
if (fps < VIDEO_TRANSCODING_FPS.MIN) {
|
||||
throw new Error(`Cannot compute FPS because ${fps} is lower than our minimum value ${VIDEO_TRANSCODING_FPS.MIN}`)
|
||||
}
|
||||
|
||||
return fps
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
// Re export ffprobe utils
|
||||
getVideoStreamDimensionsInfo,
|
||||
buildFileMetadata,
|
||||
getMaxAudioBitrate,
|
||||
getVideoStream,
|
||||
getVideoStreamDuration,
|
||||
getAudioStream,
|
||||
hasAudioStream,
|
||||
getVideoStreamFPS,
|
||||
ffprobePromise,
|
||||
getVideoStreamBitrate,
|
||||
|
||||
getVideoStreamCodec,
|
||||
getAudioStreamCodec,
|
||||
|
||||
computeFPS,
|
||||
getClosestFramerateStandard,
|
||||
|
||||
computeResolutionsToTranscode,
|
||||
|
||||
canDoQuickTranscode,
|
||||
canDoQuickVideoTranscode,
|
||||
canDoQuickAudioTranscode
|
||||
}
|
44
server/helpers/ffmpeg/framerate.ts
Normal file
44
server/helpers/ffmpeg/framerate.ts
Normal file
@ -0,0 +1,44 @@
|
||||
import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
|
||||
import { VideoResolution } from '@shared/models'
|
||||
|
||||
export function computeOutputFPS (options: {
|
||||
inputFPS: number
|
||||
resolution: VideoResolution
|
||||
}) {
|
||||
const { resolution } = options
|
||||
|
||||
let fps = options.inputFPS
|
||||
|
||||
if (
|
||||
// On small/medium resolutions, limit FPS
|
||||
resolution !== undefined &&
|
||||
resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN &&
|
||||
fps > VIDEO_TRANSCODING_FPS.AVERAGE
|
||||
) {
|
||||
// Get closest standard framerate by modulo: downsampling has to be done to a divisor of the nominal fps value
|
||||
fps = getClosestFramerateStandard({ fps, type: 'STANDARD' })
|
||||
}
|
||||
|
||||
// Hard FPS limits
|
||||
if (fps > VIDEO_TRANSCODING_FPS.MAX) fps = getClosestFramerateStandard({ fps, type: 'HD_STANDARD' })
|
||||
|
||||
if (fps < VIDEO_TRANSCODING_FPS.MIN) {
|
||||
throw new Error(`Cannot compute FPS because ${fps} is lower than our minimum value ${VIDEO_TRANSCODING_FPS.MIN}`)
|
||||
}
|
||||
|
||||
return fps
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Private
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function getClosestFramerateStandard (options: {
|
||||
fps: number
|
||||
type: 'HD_STANDARD' | 'STANDARD'
|
||||
}) {
|
||||
const { fps, type } = options
|
||||
|
||||
return VIDEO_TRANSCODING_FPS[type].slice(0)
|
||||
.sort((a, b) => fps % a - fps % b)[0]
|
||||
}
|
@ -1,8 +1,4 @@
|
||||
export * from './ffmpeg-commons'
|
||||
export * from './ffmpeg-edition'
|
||||
export * from './ffmpeg-encoders'
|
||||
export * from './ffmpeg-images'
|
||||
export * from './ffmpeg-live'
|
||||
export * from './ffmpeg-presets'
|
||||
export * from './ffmpeg-vod'
|
||||
export * from './ffprobe-utils'
|
||||
export * from './codecs'
|
||||
export * from './ffmpeg-image'
|
||||
export * from './ffmpeg-options'
|
||||
export * from './framerate'
|
||||
|
@ -3,7 +3,7 @@ import Jimp, { read as jimpRead } from 'jimp'
|
||||
import { join } from 'path'
|
||||
import { getLowercaseExtension } from '@shared/core-utils'
|
||||
import { buildUUID } from '@shared/extra-utils'
|
||||
import { convertWebPToJPG, generateThumbnailFromVideo, processGIF } from './ffmpeg/ffmpeg-images'
|
||||
import { convertWebPToJPG, generateThumbnailFromVideo, processGIF } from './ffmpeg'
|
||||
import { logger, loggerTagsFactory } from './logger'
|
||||
|
||||
const lTags = loggerTagsFactory('image-utils')
|
||||
@ -30,7 +30,7 @@ async function processImage (options: {
|
||||
|
||||
// Use FFmpeg to process GIF
|
||||
if (extension === '.gif') {
|
||||
await processGIF(path, destination, newSize)
|
||||
await processGIF({ path, destination, newSize })
|
||||
} else {
|
||||
await jimpProcessor(path, destination, newSize, extension)
|
||||
}
|
||||
@ -50,7 +50,7 @@ async function generateImageFromVideoFile (options: {
|
||||
const pendingImagePath = join(folder, pendingImageName)
|
||||
|
||||
try {
|
||||
await generateThumbnailFromVideo(fromPath, folder, imageName)
|
||||
await generateThumbnailFromVideo({ fromPath, folder, imageName })
|
||||
|
||||
const destination = join(folder, imageName)
|
||||
await processImage({ path: pendingImagePath, destination, newSize: size })
|
||||
@ -99,7 +99,7 @@ async function jimpProcessor (path: string, destination: string, newSize: { widt
|
||||
logger.debug('Cannot read %s with jimp. Try to convert the image using ffmpeg first.', path, { err })
|
||||
|
||||
const newName = path + '.jpg'
|
||||
await convertWebPToJPG(path, newName)
|
||||
await convertWebPToJPG({ path, destination: newName })
|
||||
await rename(newName, path)
|
||||
|
||||
sourceImage = await jimpRead(path)
|
||||
|
@ -2,10 +2,11 @@ import { compare, genSalt, hash } from 'bcrypt'
|
||||
import { createCipheriv, createDecipheriv, createSign, createVerify } from 'crypto'
|
||||
import { Request } from 'express'
|
||||
import { cloneDeep } from 'lodash'
|
||||
import { promisify1, promisify2 } from '@shared/core-utils'
|
||||
import { sha256 } from '@shared/extra-utils'
|
||||
import { BCRYPT_SALT_SIZE, ENCRYPTION, HTTP_SIGNATURE, PRIVATE_RSA_KEY_SIZE } from '../initializers/constants'
|
||||
import { MActor } from '../types/models'
|
||||
import { generateRSAKeyPairPromise, promisify1, promisify2, randomBytesPromise, scryptPromise } from './core-utils'
|
||||
import { generateRSAKeyPairPromise, randomBytesPromise, scryptPromise } from './core-utils'
|
||||
import { jsonld } from './custom-jsonld-signature'
|
||||
import { logger } from './logger'
|
||||
|
||||
|
19
server/helpers/token-generator.ts
Normal file
19
server/helpers/token-generator.ts
Normal file
@ -0,0 +1,19 @@
|
||||
import { buildUUID } from '@shared/extra-utils'
|
||||
|
||||
function generateRunnerRegistrationToken () {
|
||||
return 'ptrrt-' + buildUUID()
|
||||
}
|
||||
|
||||
function generateRunnerToken () {
|
||||
return 'ptrt-' + buildUUID()
|
||||
}
|
||||
|
||||
function generateRunnerJobToken () {
|
||||
return 'ptrjt-' + buildUUID()
|
||||
}
|
||||
|
||||
export {
|
||||
generateRunnerRegistrationToken,
|
||||
generateRunnerToken,
|
||||
generateRunnerJobToken
|
||||
}
|
@ -13,9 +13,9 @@ import { VideoPathManager } from '@server/lib/video-path-manager'
|
||||
import { MVideo } from '@server/types/models/video/video'
|
||||
import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file'
|
||||
import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist'
|
||||
import { promisify2 } from '@shared/core-utils'
|
||||
import { sha1 } from '@shared/extra-utils'
|
||||
import { CONFIG } from '../initializers/config'
|
||||
import { promisify2 } from './core-utils'
|
||||
import { logger } from './logger'
|
||||
import { generateVideoImportTmpPath } from './utils'
|
||||
import { extractVideo } from './video'
|
||||
|
@ -1,7 +1,7 @@
|
||||
import config from 'config'
|
||||
import { URL } from 'url'
|
||||
import { getFFmpegVersion } from '@server/helpers/ffmpeg'
|
||||
import { uniqify } from '@shared/core-utils'
|
||||
import { getFFmpegVersion } from '@shared/ffmpeg'
|
||||
import { VideoRedundancyConfigFilter } from '@shared/models/redundancy/video-redundancy-config-filter.type'
|
||||
import { RecentlyAddedStrategy } from '../../shared/models/redundancy'
|
||||
import { isProdInstance, parseBytes, parseSemVersion } from '../helpers/core-utils'
|
||||
|
@ -1,5 +1,6 @@
|
||||
import { IConfig } from 'config'
|
||||
import { parseSemVersion, promisify0 } from '../helpers/core-utils'
|
||||
import { promisify0 } from '@shared/core-utils'
|
||||
import { parseSemVersion } from '../helpers/core-utils'
|
||||
import { logger } from '../helpers/logger'
|
||||
|
||||
// Special behaviour for config because we can reload it
|
||||
@ -36,7 +37,9 @@ function checkMissedConfig () {
|
||||
'transcoding.profile', 'transcoding.concurrency',
|
||||
'transcoding.resolutions.0p', 'transcoding.resolutions.144p', 'transcoding.resolutions.240p', 'transcoding.resolutions.360p',
|
||||
'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p',
|
||||
'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'video_studio.enabled',
|
||||
'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'transcoding.remote_runners.enabled',
|
||||
'video_studio.enabled',
|
||||
'remote_runners.stalled_jobs.vod', 'remote_runners.stalled_jobs.live',
|
||||
'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout',
|
||||
'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user',
|
||||
'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization',
|
||||
@ -74,7 +77,8 @@ function checkMissedConfig () {
|
||||
'live.transcoding.enabled', 'live.transcoding.threads', 'live.transcoding.profile',
|
||||
'live.transcoding.resolutions.144p', 'live.transcoding.resolutions.240p', 'live.transcoding.resolutions.360p',
|
||||
'live.transcoding.resolutions.480p', 'live.transcoding.resolutions.720p', 'live.transcoding.resolutions.1080p',
|
||||
'live.transcoding.resolutions.1440p', 'live.transcoding.resolutions.2160p', 'live.transcoding.always_transcode_original_resolution'
|
||||
'live.transcoding.resolutions.1440p', 'live.transcoding.resolutions.2160p', 'live.transcoding.always_transcode_original_resolution',
|
||||
'live.transcoding.remote_runners.enabled'
|
||||
]
|
||||
|
||||
const requiredAlternatives = [
|
||||
|
@ -304,6 +304,12 @@ const CONFIG = {
|
||||
COUNT: config.get<number>('feeds.comments.count')
|
||||
}
|
||||
},
|
||||
REMOTE_RUNNERS: {
|
||||
STALLED_JOBS: {
|
||||
LIVE: parseDurationToMs(config.get<string>('remote_runners.stalled_jobs.live')),
|
||||
VOD: parseDurationToMs(config.get<string>('remote_runners.stalled_jobs.vod'))
|
||||
}
|
||||
},
|
||||
ADMIN: {
|
||||
get EMAIL () { return config.get<string>('admin.email') }
|
||||
},
|
||||
@ -359,6 +365,9 @@ const CONFIG = {
|
||||
},
|
||||
WEBTORRENT: {
|
||||
get ENABLED () { return config.get<boolean>('transcoding.webtorrent.enabled') }
|
||||
},
|
||||
REMOTE_RUNNERS: {
|
||||
get ENABLED () { return config.get<boolean>('transcoding.remote_runners.enabled') }
|
||||
}
|
||||
},
|
||||
LIVE: {
|
||||
@ -406,6 +415,9 @@ const CONFIG = {
|
||||
get '1080p' () { return config.get<boolean>('live.transcoding.resolutions.1080p') },
|
||||
get '1440p' () { return config.get<boolean>('live.transcoding.resolutions.1440p') },
|
||||
get '2160p' () { return config.get<boolean>('live.transcoding.resolutions.2160p') }
|
||||
},
|
||||
REMOTE_RUNNERS: {
|
||||
get ENABLED () { return config.get<boolean>('live.transcoding.remote_runners.enabled') }
|
||||
}
|
||||
}
|
||||
},
|
||||
|
@ -6,6 +6,7 @@ import { randomInt, root } from '@shared/core-utils'
|
||||
import {
|
||||
AbuseState,
|
||||
JobType,
|
||||
RunnerJobState,
|
||||
UserRegistrationState,
|
||||
VideoChannelSyncState,
|
||||
VideoImportState,
|
||||
@ -26,7 +27,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const LAST_MIGRATION_VERSION = 760
|
||||
const LAST_MIGRATION_VERSION = 765
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@ -81,6 +82,10 @@ const SORTABLE_COLUMNS = {
|
||||
|
||||
USER_REGISTRATIONS: [ 'createdAt', 'state' ],
|
||||
|
||||
RUNNERS: [ 'createdAt' ],
|
||||
RUNNER_REGISTRATION_TOKENS: [ 'createdAt' ],
|
||||
RUNNER_JOBS: [ 'updatedAt', 'createdAt', 'priority', 'state', 'progress' ],
|
||||
|
||||
VIDEOS: [ 'name', 'duration', 'createdAt', 'publishedAt', 'originallyPublishedAt', 'views', 'likes', 'trending', 'hot', 'best' ],
|
||||
|
||||
// Don't forget to update peertube-search-index with the same values
|
||||
@ -139,6 +144,8 @@ const REMOTE_SCHEME = {
|
||||
WS: 'wss'
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const JOB_ATTEMPTS: { [id in JobType]: number } = {
|
||||
'activitypub-http-broadcast': 1,
|
||||
'activitypub-http-broadcast-parallel': 1,
|
||||
@ -160,6 +167,7 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
|
||||
'video-channel-import': 1,
|
||||
'after-video-channel-import': 1,
|
||||
'move-to-object-storage': 3,
|
||||
'transcoding-job-builder': 1,
|
||||
'notify': 1,
|
||||
'federate-video': 1
|
||||
}
|
||||
@ -183,6 +191,7 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im
|
||||
'move-to-object-storage': 1,
|
||||
'video-channel-import': 1,
|
||||
'after-video-channel-import': 1,
|
||||
'transcoding-job-builder': 1,
|
||||
'notify': 5,
|
||||
'federate-video': 3
|
||||
}
|
||||
@ -207,6 +216,7 @@ const JOB_TTL: { [id in JobType]: number } = {
|
||||
'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours
|
||||
'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours
|
||||
'after-video-channel-import': 60000 * 5, // 5 minutes
|
||||
'transcoding-job-builder': 60000, // 1 minute
|
||||
'notify': 60000 * 5, // 5 minutes
|
||||
'federate-video': 60000 * 5 // 5 minutes
|
||||
}
|
||||
@ -222,21 +232,6 @@ const JOB_PRIORITY = {
|
||||
TRANSCODING: 100
|
||||
}
|
||||
|
||||
const BROADCAST_CONCURRENCY = 30 // How many requests in parallel we do in activitypub-http-broadcast job
|
||||
const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...)
|
||||
|
||||
const AP_CLEANER = {
|
||||
CONCURRENCY: 10, // How many requests in parallel we do in activitypub-cleaner job
|
||||
UNAVAILABLE_TRESHOLD: 3, // How many attempts we do before removing an unavailable remote resource
|
||||
PERIOD: parseDurationToMs('1 week') // /!\ Has to be sync with REPEAT_JOBS
|
||||
}
|
||||
|
||||
const REQUEST_TIMEOUTS = {
|
||||
DEFAULT: 7000, // 7 seconds
|
||||
FILE: 30000, // 30 seconds
|
||||
REDUNDANCY: JOB_TTL['video-redundancy']
|
||||
}
|
||||
|
||||
const JOB_REMOVAL_OPTIONS = {
|
||||
COUNT: 10000, // Max jobs to store
|
||||
|
||||
@ -256,7 +251,29 @@ const JOB_REMOVAL_OPTIONS = {
|
||||
|
||||
const VIDEO_IMPORT_TIMEOUT = Math.floor(JOB_TTL['video-import'] * 0.9)
|
||||
|
||||
const RUNNER_JOBS = {
|
||||
MAX_FAILURES: 5
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const BROADCAST_CONCURRENCY = 30 // How many requests in parallel we do in activitypub-http-broadcast job
|
||||
const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...)
|
||||
|
||||
const AP_CLEANER = {
|
||||
CONCURRENCY: 10, // How many requests in parallel we do in activitypub-cleaner job
|
||||
UNAVAILABLE_TRESHOLD: 3, // How many attempts we do before removing an unavailable remote resource
|
||||
PERIOD: parseDurationToMs('1 week') // /!\ Has to be sync with REPEAT_JOBS
|
||||
}
|
||||
|
||||
const REQUEST_TIMEOUTS = {
|
||||
DEFAULT: 7000, // 7 seconds
|
||||
FILE: 30000, // 30 seconds
|
||||
REDUNDANCY: JOB_TTL['video-redundancy']
|
||||
}
|
||||
|
||||
const SCHEDULER_INTERVALS_MS = {
|
||||
RUNNER_JOB_WATCH_DOG: Math.min(CONFIG.REMOTE_RUNNERS.STALLED_JOBS.VOD, CONFIG.REMOTE_RUNNERS.STALLED_JOBS.LIVE),
|
||||
ACTOR_FOLLOW_SCORES: 60000 * 60, // 1 hour
|
||||
REMOVE_OLD_JOBS: 60000 * 60, // 1 hour
|
||||
UPDATE_VIDEOS: 60000, // 1 minute
|
||||
@ -410,6 +427,17 @@ const CONSTRAINTS_FIELDS = {
|
||||
CLIENT_STACK_TRACE: { min: 1, max: 15000 }, // Length
|
||||
CLIENT_META: { min: 1, max: 5000 }, // Length
|
||||
CLIENT_USER_AGENT: { min: 1, max: 200 } // Length
|
||||
},
|
||||
RUNNERS: {
|
||||
TOKEN: { min: 1, max: 1000 }, // Length
|
||||
NAME: { min: 1, max: 100 }, // Length
|
||||
DESCRIPTION: { min: 1, max: 1000 } // Length
|
||||
},
|
||||
RUNNER_JOBS: {
|
||||
TOKEN: { min: 1, max: 1000 }, // Length
|
||||
REASON: { min: 1, max: 5000 }, // Length
|
||||
ERROR_MESSAGE: { min: 1, max: 5000 }, // Length
|
||||
PROGRESS: { min: 0, max: 100 } // Value
|
||||
}
|
||||
}
|
||||
|
||||
@ -540,6 +568,17 @@ const VIDEO_PLAYLIST_TYPES: { [ id in VideoPlaylistType ]: string } = {
|
||||
[VideoPlaylistType.WATCH_LATER]: 'Watch later'
|
||||
}
|
||||
|
||||
const RUNNER_JOB_STATES: { [ id in RunnerJobState ]: string } = {
|
||||
[RunnerJobState.PROCESSING]: 'Processing',
|
||||
[RunnerJobState.COMPLETED]: 'Completed',
|
||||
[RunnerJobState.PENDING]: 'Pending',
|
||||
[RunnerJobState.ERRORED]: 'Errored',
|
||||
[RunnerJobState.WAITING_FOR_PARENT_JOB]: 'Waiting for parent job to finish',
|
||||
[RunnerJobState.CANCELLED]: 'Cancelled',
|
||||
[RunnerJobState.PARENT_ERRORED]: 'Parent job failed',
|
||||
[RunnerJobState.PARENT_CANCELLED]: 'Parent job cancelled'
|
||||
}
|
||||
|
||||
const MIMETYPES = {
|
||||
AUDIO: {
|
||||
MIMETYPE_EXT: {
|
||||
@ -594,6 +633,11 @@ const MIMETYPES = {
|
||||
MIMETYPE_EXT: {
|
||||
'application/x-bittorrent': '.torrent'
|
||||
}
|
||||
},
|
||||
M3U8: {
|
||||
MIMETYPE_EXT: {
|
||||
'application/vnd.apple.mpegurl': '.m3u8'
|
||||
}
|
||||
}
|
||||
}
|
||||
MIMETYPES.AUDIO.EXT_MIMETYPE = invert(MIMETYPES.AUDIO.MIMETYPE_EXT)
|
||||
@ -1027,6 +1071,7 @@ export {
|
||||
SEARCH_INDEX,
|
||||
DIRECTORIES,
|
||||
RESUMABLE_UPLOAD_SESSION_LIFETIME,
|
||||
RUNNER_JOB_STATES,
|
||||
P2P_MEDIA_LOADER_PEER_VERSION,
|
||||
ACTOR_IMAGES_SIZE,
|
||||
ACCEPT_HEADERS,
|
||||
@ -1085,6 +1130,7 @@ export {
|
||||
USER_REGISTRATION_STATES,
|
||||
LRU_CACHE,
|
||||
REQUEST_TIMEOUTS,
|
||||
RUNNER_JOBS,
|
||||
MAX_LOCAL_VIEWER_WATCH_SECTIONS,
|
||||
USER_PASSWORD_RESET_LIFETIME,
|
||||
USER_PASSWORD_CREATE_LIFETIME,
|
||||
|
@ -1,6 +1,9 @@
|
||||
import { QueryTypes, Transaction } from 'sequelize'
|
||||
import { Sequelize as SequelizeTypescript } from 'sequelize-typescript'
|
||||
import { ActorCustomPageModel } from '@server/models/account/actor-custom-page'
|
||||
import { RunnerModel } from '@server/models/runner/runner'
|
||||
import { RunnerJobModel } from '@server/models/runner/runner-job'
|
||||
import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token'
|
||||
import { TrackerModel } from '@server/models/server/tracker'
|
||||
import { VideoTrackerModel } from '@server/models/server/video-tracker'
|
||||
import { UserModel } from '@server/models/user/user'
|
||||
@ -9,6 +12,7 @@ import { UserRegistrationModel } from '@server/models/user/user-registration'
|
||||
import { UserVideoHistoryModel } from '@server/models/user/user-video-history'
|
||||
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
|
||||
import { VideoJobInfoModel } from '@server/models/video/video-job-info'
|
||||
import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting'
|
||||
import { VideoLiveSessionModel } from '@server/models/video/video-live-session'
|
||||
import { VideoSourceModel } from '@server/models/video/video-source'
|
||||
import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer'
|
||||
@ -52,7 +56,6 @@ import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-pla
|
||||
import { VideoTagModel } from '../models/video/video-tag'
|
||||
import { VideoViewModel } from '../models/view/video-view'
|
||||
import { CONFIG } from './config'
|
||||
import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting'
|
||||
|
||||
require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
|
||||
|
||||
@ -159,7 +162,10 @@ async function initDatabaseModels (silent: boolean) {
|
||||
ActorCustomPageModel,
|
||||
VideoJobInfoModel,
|
||||
VideoChannelSyncModel,
|
||||
UserRegistrationModel
|
||||
UserRegistrationModel,
|
||||
RunnerRegistrationTokenModel,
|
||||
RunnerModel,
|
||||
RunnerJobModel
|
||||
])
|
||||
|
||||
// Check extensions exist in the database
|
||||
|
@ -2,7 +2,9 @@ import { ensureDir, readdir, remove } from 'fs-extra'
|
||||
import passwordGenerator from 'password-generator'
|
||||
import { join } from 'path'
|
||||
import { isTestOrDevInstance } from '@server/helpers/core-utils'
|
||||
import { generateRunnerRegistrationToken } from '@server/helpers/token-generator'
|
||||
import { getNodeABIVersion } from '@server/helpers/version'
|
||||
import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token'
|
||||
import { UserRole } from '@shared/models'
|
||||
import { logger } from '../helpers/logger'
|
||||
import { buildUser, createApplicationActor, createUserAccountAndChannelAndPlaylist } from '../lib/user'
|
||||
@ -22,7 +24,8 @@ async function installApplication () {
|
||||
return Promise.all([
|
||||
createApplicationIfNotExist(),
|
||||
createOAuthClientIfNotExist(),
|
||||
createOAuthAdminIfNotExist()
|
||||
createOAuthAdminIfNotExist(),
|
||||
createRunnerRegistrationTokenIfNotExist()
|
||||
])
|
||||
}),
|
||||
|
||||
@ -183,3 +186,14 @@ async function createApplicationIfNotExist () {
|
||||
|
||||
return createApplicationActor(application.id)
|
||||
}
|
||||
|
||||
async function createRunnerRegistrationTokenIfNotExist () {
|
||||
const total = await RunnerRegistrationTokenModel.countTotal()
|
||||
if (total !== 0) return undefined
|
||||
|
||||
const token = new RunnerRegistrationTokenModel({
|
||||
registrationToken: generateRunnerRegistrationToken()
|
||||
})
|
||||
|
||||
await token.save()
|
||||
}
|
||||
|
78
server/initializers/migrations/0765-remote-transcoding.ts
Normal file
78
server/initializers/migrations/0765-remote-transcoding.ts
Normal file
@ -0,0 +1,78 @@
|
||||
import * as Sequelize from 'sequelize'
|
||||
|
||||
async function up (utils: {
|
||||
transaction: Sequelize.Transaction
|
||||
queryInterface: Sequelize.QueryInterface
|
||||
sequelize: Sequelize.Sequelize
|
||||
}): Promise<void> {
|
||||
{
|
||||
const query = `
|
||||
CREATE TABLE IF NOT EXISTS "runnerRegistrationToken"(
|
||||
"id" serial,
|
||||
"registrationToken" varchar(255) NOT NULL,
|
||||
"createdAt" timestamp with time zone NOT NULL,
|
||||
"updatedAt" timestamp with time zone NOT NULL,
|
||||
PRIMARY KEY ("id")
|
||||
);
|
||||
`
|
||||
|
||||
await utils.sequelize.query(query, { transaction : utils.transaction })
|
||||
}
|
||||
|
||||
{
|
||||
const query = `
|
||||
CREATE TABLE IF NOT EXISTS "runner"(
|
||||
"id" serial,
|
||||
"runnerToken" varchar(255) NOT NULL,
|
||||
"name" varchar(255) NOT NULL,
|
||||
"description" varchar(1000),
|
||||
"lastContact" timestamp with time zone NOT NULL,
|
||||
"ip" varchar(255) NOT NULL,
|
||||
"runnerRegistrationTokenId" integer REFERENCES "runnerRegistrationToken"("id") ON DELETE CASCADE ON UPDATE CASCADE,
|
||||
"createdAt" timestamp with time zone NOT NULL,
|
||||
"updatedAt" timestamp with time zone NOT NULL,
|
||||
PRIMARY KEY ("id")
|
||||
);
|
||||
`
|
||||
|
||||
await utils.sequelize.query(query, { transaction : utils.transaction })
|
||||
}
|
||||
|
||||
{
|
||||
const query = `
|
||||
CREATE TABLE IF NOT EXISTS "runnerJob"(
|
||||
"id" serial,
|
||||
"uuid" uuid NOT NULL,
|
||||
"type" varchar(255) NOT NULL,
|
||||
"payload" jsonb NOT NULL,
|
||||
"privatePayload" jsonb NOT NULL,
|
||||
"state" integer NOT NULL,
|
||||
"failures" integer NOT NULL DEFAULT 0,
|
||||
"error" varchar(5000),
|
||||
"priority" integer NOT NULL,
|
||||
"processingJobToken" varchar(255),
|
||||
"progress" integer,
|
||||
"startedAt" timestamp with time zone,
|
||||
"finishedAt" timestamp with time zone,
|
||||
"dependsOnRunnerJobId" integer REFERENCES "runnerJob"("id") ON DELETE CASCADE ON UPDATE CASCADE,
|
||||
"runnerId" integer REFERENCES "runner"("id") ON DELETE SET NULL ON UPDATE CASCADE,
|
||||
"createdAt" timestamp with time zone NOT NULL,
|
||||
"updatedAt" timestamp with time zone NOT NULL,
|
||||
PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
|
||||
`
|
||||
|
||||
await utils.sequelize.query(query, { transaction : utils.transaction })
|
||||
}
|
||||
}
|
||||
|
||||
function down (options) {
|
||||
throw new Error('Not implemented.')
|
||||
}
|
||||
|
||||
export {
|
||||
up,
|
||||
down
|
||||
}
|
@ -3,10 +3,11 @@ import { flatten } from 'lodash'
|
||||
import PQueue from 'p-queue'
|
||||
import { basename, dirname, join } from 'path'
|
||||
import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models'
|
||||
import { uniqify } from '@shared/core-utils'
|
||||
import { uniqify, uuidRegex } from '@shared/core-utils'
|
||||
import { sha256 } from '@shared/extra-utils'
|
||||
import { getVideoStreamDimensionsInfo } from '@shared/ffmpeg'
|
||||
import { VideoStorage } from '@shared/models'
|
||||
import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamDimensionsInfo } from '../helpers/ffmpeg'
|
||||
import { getAudioStreamCodec, getVideoStreamCodec } from '../helpers/ffmpeg'
|
||||
import { logger } from '../helpers/logger'
|
||||
import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
|
||||
import { generateRandomString } from '../helpers/utils'
|
||||
@ -234,6 +235,16 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string,
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function renameVideoFileInPlaylist (playlistPath: string, newVideoFilename: string) {
|
||||
const content = await readFile(playlistPath, 'utf8')
|
||||
|
||||
const newContent = content.replace(new RegExp(`${uuidRegex}-\\d+-fragmented.mp4`, 'g'), newVideoFilename)
|
||||
|
||||
await writeFile(playlistPath, newContent, 'utf8')
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function injectQueryToPlaylistUrls (content: string, queryString: string) {
|
||||
return content.replace(/\.(m3u8|ts|mp4)/gm, '.$1?' + queryString)
|
||||
}
|
||||
@ -247,7 +258,8 @@ export {
|
||||
downloadPlaylistSegments,
|
||||
updateStreamingPlaylistsInfohashesIfNeeded,
|
||||
updatePlaylistAfterFileChange,
|
||||
injectQueryToPlaylistUrls
|
||||
injectQueryToPlaylistUrls,
|
||||
renameVideoFileInPlaylist
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
47
server/lib/job-queue/handlers/transcoding-job-builder.ts
Normal file
47
server/lib/job-queue/handlers/transcoding-job-builder.ts
Normal file
@ -0,0 +1,47 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job'
|
||||
import { UserModel } from '@server/models/user/user'
|
||||
import { VideoModel } from '@server/models/video/video'
|
||||
import { VideoJobInfoModel } from '@server/models/video/video-job-info'
|
||||
import { pick } from '@shared/core-utils'
|
||||
import { TranscodingJobBuilderPayload } from '@shared/models'
|
||||
import { logger } from '../../../helpers/logger'
|
||||
import { JobQueue } from '../job-queue'
|
||||
|
||||
async function processTranscodingJobBuilder (job: Job) {
|
||||
const payload = job.data as TranscodingJobBuilderPayload
|
||||
|
||||
logger.info('Processing transcoding job builder in job %s.', job.id)
|
||||
|
||||
if (payload.optimizeJob) {
|
||||
const video = await VideoModel.loadFull(payload.videoUUID)
|
||||
const user = await UserModel.loadByVideoId(video.id)
|
||||
const videoFile = video.getMaxQualityFile()
|
||||
|
||||
await createOptimizeOrMergeAudioJobs({
|
||||
...pick(payload.optimizeJob, [ 'isNewVideo' ]),
|
||||
|
||||
video,
|
||||
videoFile,
|
||||
user
|
||||
})
|
||||
}
|
||||
|
||||
for (const job of (payload.jobs || [])) {
|
||||
await JobQueue.Instance.createJob(job)
|
||||
|
||||
await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode')
|
||||
}
|
||||
|
||||
for (const sequentialJobs of (payload.sequentialJobs || [])) {
|
||||
await JobQueue.Instance.createSequentialJobFlow(...sequentialJobs)
|
||||
|
||||
await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode', sequentialJobs.length)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processTranscodingJobBuilder
|
||||
}
|
@ -10,8 +10,8 @@ import { VideoModel } from '@server/models/video/video'
|
||||
import { VideoFileModel } from '@server/models/video/video-file'
|
||||
import { MVideoFullLight } from '@server/types/models'
|
||||
import { getLowercaseExtension } from '@shared/core-utils'
|
||||
import { getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg'
|
||||
import { VideoFileImportPayload, VideoStorage } from '@shared/models'
|
||||
import { getVideoStreamFPS, getVideoStreamDimensionsInfo } from '../../../helpers/ffmpeg'
|
||||
import { logger } from '../../../helpers/logger'
|
||||
import { JobQueue } from '../job-queue'
|
||||
|
||||
|
@ -7,15 +7,16 @@ import { isPostImportVideoAccepted } from '@server/lib/moderation'
|
||||
import { generateWebTorrentVideoFilename } from '@server/lib/paths'
|
||||
import { Hooks } from '@server/lib/plugins/hooks'
|
||||
import { ServerConfigManager } from '@server/lib/server-config-manager'
|
||||
import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job'
|
||||
import { isAbleToUploadVideo } from '@server/lib/user'
|
||||
import { buildMoveToObjectStorageJob, buildOptimizeOrMergeAudioJob } from '@server/lib/video'
|
||||
import { buildMoveToObjectStorageJob } from '@server/lib/video'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager'
|
||||
import { buildNextVideoState } from '@server/lib/video-state'
|
||||
import { ThumbnailModel } from '@server/models/video/thumbnail'
|
||||
import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models'
|
||||
import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import'
|
||||
import { getLowercaseExtension } from '@shared/core-utils'
|
||||
import { isAudioFile } from '@shared/extra-utils'
|
||||
import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS, isAudioFile } from '@shared/ffmpeg'
|
||||
import {
|
||||
ThumbnailType,
|
||||
VideoImportPayload,
|
||||
@ -28,7 +29,6 @@ import {
|
||||
VideoResolution,
|
||||
VideoState
|
||||
} from '@shared/models'
|
||||
import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS } from '../../../helpers/ffmpeg'
|
||||
import { logger } from '../../../helpers/logger'
|
||||
import { getSecureTorrentName } from '../../../helpers/utils'
|
||||
import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
|
||||
@ -137,7 +137,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
|
||||
|
||||
const { resolution } = await isAudioFile(tempVideoPath, probe)
|
||||
? { resolution: VideoResolution.H_NOVIDEO }
|
||||
: await getVideoStreamDimensionsInfo(tempVideoPath)
|
||||
: await getVideoStreamDimensionsInfo(tempVideoPath, probe)
|
||||
|
||||
const fps = await getVideoStreamFPS(tempVideoPath, probe)
|
||||
const duration = await getVideoStreamDuration(tempVideoPath, probe)
|
||||
@ -313,9 +313,7 @@ async function afterImportSuccess (options: {
|
||||
}
|
||||
|
||||
if (video.state === VideoState.TO_TRANSCODE) { // Create transcoding jobs?
|
||||
await JobQueue.Instance.createJob(
|
||||
await buildOptimizeOrMergeAudioJob({ video, videoFile, user })
|
||||
)
|
||||
await createOptimizeOrMergeAudioJobs({ video, videoFile, isNewVideo: true, user })
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1,25 +1,25 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { readdir, remove } from 'fs-extra'
|
||||
import { join } from 'path'
|
||||
import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo } from '@server/helpers/ffmpeg'
|
||||
import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
|
||||
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
|
||||
import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live'
|
||||
import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths'
|
||||
import { generateVideoMiniature } from '@server/lib/thumbnail'
|
||||
import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/transcoding'
|
||||
import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/hls-transcoding'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager'
|
||||
import { moveToNextState } from '@server/lib/video-state'
|
||||
import { VideoModel } from '@server/models/video/video'
|
||||
import { VideoBlacklistModel } from '@server/models/video/video-blacklist'
|
||||
import { VideoFileModel } from '@server/models/video/video-file'
|
||||
import { VideoLiveModel } from '@server/models/video/video-live'
|
||||
import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting'
|
||||
import { VideoLiveSessionModel } from '@server/models/video/video-live-session'
|
||||
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
|
||||
import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@server/types/models'
|
||||
import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg'
|
||||
import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models'
|
||||
import { logger, loggerTagsFactory } from '../../../helpers/logger'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager'
|
||||
import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting'
|
||||
|
||||
const lTags = loggerTagsFactory('live', 'job')
|
||||
|
||||
@ -224,6 +224,7 @@ async function assignReplayFilesToVideo (options: {
|
||||
const probe = await ffprobePromise(concatenatedTsFilePath)
|
||||
const { audioStream } = await getAudioStream(concatenatedTsFilePath, probe)
|
||||
const { resolution } = await getVideoStreamDimensionsInfo(concatenatedTsFilePath, probe)
|
||||
const fps = await getVideoStreamFPS(concatenatedTsFilePath, probe)
|
||||
|
||||
try {
|
||||
await generateHlsPlaylistResolutionFromTS({
|
||||
@ -231,6 +232,7 @@ async function assignReplayFilesToVideo (options: {
|
||||
inputFileMutexReleaser,
|
||||
concatenatedTsFilePath,
|
||||
resolution,
|
||||
fps,
|
||||
isAAC: audioStream?.codec_name === 'aac'
|
||||
})
|
||||
} catch (err) {
|
||||
|
@ -1,15 +1,16 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { move, remove } from 'fs-extra'
|
||||
import { join } from 'path'
|
||||
import { addIntroOutro, addWatermark, cutVideo } from '@server/helpers/ffmpeg'
|
||||
import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg'
|
||||
import { createTorrentAndSetInfoHashFromPath } from '@server/helpers/webtorrent'
|
||||
import { CONFIG } from '@server/initializers/config'
|
||||
import { VIDEO_FILTERS } from '@server/initializers/constants'
|
||||
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
|
||||
import { generateWebTorrentVideoFilename } from '@server/lib/paths'
|
||||
import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job'
|
||||
import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles'
|
||||
import { isAbleToUploadVideo } from '@server/lib/user'
|
||||
import { buildOptimizeOrMergeAudioJob } from '@server/lib/video'
|
||||
import { removeHLSPlaylist, removeWebTorrentFile } from '@server/lib/video-file'
|
||||
import { buildFileMetadata, removeHLSPlaylist, removeWebTorrentFile } from '@server/lib/video-file'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager'
|
||||
import { approximateIntroOutroAdditionalSize } from '@server/lib/video-studio'
|
||||
import { UserModel } from '@server/models/user/user'
|
||||
@ -17,15 +18,8 @@ import { VideoModel } from '@server/models/video/video'
|
||||
import { VideoFileModel } from '@server/models/video/video-file'
|
||||
import { MVideo, MVideoFile, MVideoFullLight, MVideoId, MVideoWithAllFiles } from '@server/types/models'
|
||||
import { getLowercaseExtension, pick } from '@shared/core-utils'
|
||||
import {
|
||||
buildFileMetadata,
|
||||
buildUUID,
|
||||
ffprobePromise,
|
||||
getFileSize,
|
||||
getVideoStreamDimensionsInfo,
|
||||
getVideoStreamDuration,
|
||||
getVideoStreamFPS
|
||||
} from '@shared/extra-utils'
|
||||
import { buildUUID, getFileSize } from '@shared/extra-utils'
|
||||
import { FFmpegEdition, ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS } from '@shared/ffmpeg'
|
||||
import {
|
||||
VideoStudioEditionPayload,
|
||||
VideoStudioTask,
|
||||
@ -36,7 +30,6 @@ import {
|
||||
VideoStudioTaskWatermarkPayload
|
||||
} from '@shared/models'
|
||||
import { logger, loggerTagsFactory } from '../../../helpers/logger'
|
||||
import { JobQueue } from '../job-queue'
|
||||
|
||||
const lTagsBase = loggerTagsFactory('video-edition')
|
||||
|
||||
@ -102,9 +95,7 @@ async function processVideoStudioEdition (job: Job) {
|
||||
|
||||
const user = await UserModel.loadByVideoId(video.id)
|
||||
|
||||
await JobQueue.Instance.createJob(
|
||||
await buildOptimizeOrMergeAudioJob({ video, videoFile: newFile, user, isNewVideo: false })
|
||||
)
|
||||
await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user })
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@ -131,9 +122,9 @@ const taskProcessors: { [id in VideoStudioTask['name']]: (options: TaskProcessor
|
||||
}
|
||||
|
||||
async function processTask (options: TaskProcessorOptions) {
|
||||
const { video, task } = options
|
||||
const { video, task, lTags } = options
|
||||
|
||||
logger.info('Processing %s task for video %s.', task.name, video.uuid, { task, ...options.lTags })
|
||||
logger.info('Processing %s task for video %s.', task.name, video.uuid, { task, ...lTags })
|
||||
|
||||
const processor = taskProcessors[options.task.name]
|
||||
if (!process) throw new Error('Unknown task ' + task.name)
|
||||
@ -142,48 +133,53 @@ async function processTask (options: TaskProcessorOptions) {
|
||||
}
|
||||
|
||||
function processAddIntroOutro (options: TaskProcessorOptions<VideoStudioTaskIntroPayload | VideoStudioTaskOutroPayload>) {
|
||||
const { task } = options
|
||||
const { task, lTags } = options
|
||||
|
||||
return addIntroOutro({
|
||||
logger.debug('Will add intro/outro to the video.', { options, ...lTags })
|
||||
|
||||
return buildFFmpegEdition().addIntroOutro({
|
||||
...pick(options, [ 'inputPath', 'outputPath' ]),
|
||||
|
||||
introOutroPath: task.options.file,
|
||||
type: task.name === 'add-intro'
|
||||
? 'intro'
|
||||
: 'outro',
|
||||
|
||||
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
|
||||
profile: CONFIG.TRANSCODING.PROFILE
|
||||
: 'outro'
|
||||
})
|
||||
}
|
||||
|
||||
function processCut (options: TaskProcessorOptions<VideoStudioTaskCutPayload>) {
|
||||
const { task } = options
|
||||
const { task, lTags } = options
|
||||
|
||||
return cutVideo({
|
||||
logger.debug('Will cut the video.', { options, ...lTags })
|
||||
|
||||
return buildFFmpegEdition().cutVideo({
|
||||
...pick(options, [ 'inputPath', 'outputPath' ]),
|
||||
|
||||
start: task.options.start,
|
||||
end: task.options.end,
|
||||
|
||||
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
|
||||
profile: CONFIG.TRANSCODING.PROFILE
|
||||
end: task.options.end
|
||||
})
|
||||
}
|
||||
|
||||
function processAddWatermark (options: TaskProcessorOptions<VideoStudioTaskWatermarkPayload>) {
|
||||
const { task } = options
|
||||
const { task, lTags } = options
|
||||
|
||||
return addWatermark({
|
||||
logger.debug('Will add watermark to the video.', { options, ...lTags })
|
||||
|
||||
return buildFFmpegEdition().addWatermark({
|
||||
...pick(options, [ 'inputPath', 'outputPath' ]),
|
||||
|
||||
watermarkPath: task.options.file,
|
||||
|
||||
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
|
||||
profile: CONFIG.TRANSCODING.PROFILE
|
||||
videoFilters: {
|
||||
watermarkSizeRatio: VIDEO_FILTERS.WATERMARK.SIZE_RATIO,
|
||||
horitonzalMarginRatio: VIDEO_FILTERS.WATERMARK.HORIZONTAL_MARGIN_RATIO,
|
||||
verticalMarginRatio: VIDEO_FILTERS.WATERMARK.VERTICAL_MARGIN_RATIO
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function buildNewFile (video: MVideoId, path: string) {
|
||||
const videoFile = new VideoFileModel({
|
||||
extname: getLowercaseExtension(path),
|
||||
@ -223,3 +219,7 @@ async function checkUserQuotaOrThrow (video: MVideoFullLight, payload: VideoStud
|
||||
throw new Error('Quota exceeded for this user to edit the video')
|
||||
}
|
||||
}
|
||||
|
||||
function buildFFmpegEdition () {
|
||||
return new FFmpegEdition(getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders()))
|
||||
}
|
||||
|
@ -1,13 +1,13 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { TranscodeVODOptionsType } from '@server/helpers/ffmpeg'
|
||||
import { Hooks } from '@server/lib/plugins/hooks'
|
||||
import { buildTranscodingJob, getTranscodingJobPriority } from '@server/lib/video'
|
||||
import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding'
|
||||
import { generateHlsPlaylistResolution } from '@server/lib/transcoding/hls-transcoding'
|
||||
import { mergeAudioVideofile, optimizeOriginalVideofile, transcodeNewWebTorrentResolution } from '@server/lib/transcoding/web-transcoding'
|
||||
import { removeAllWebTorrentFiles } from '@server/lib/video-file'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager'
|
||||
import { moveToFailedTranscodingState, moveToNextState } from '@server/lib/video-state'
|
||||
import { moveToFailedTranscodingState } from '@server/lib/video-state'
|
||||
import { UserModel } from '@server/models/user/user'
|
||||
import { VideoJobInfoModel } from '@server/models/video/video-job-info'
|
||||
import { MUser, MUserId, MVideo, MVideoFullLight, MVideoWithFile } from '@server/types/models'
|
||||
import { pick } from '@shared/core-utils'
|
||||
import { MUser, MUserId, MVideoFullLight } from '@server/types/models'
|
||||
import {
|
||||
HLSTranscodingPayload,
|
||||
MergeAudioTranscodingPayload,
|
||||
@ -15,18 +15,8 @@ import {
|
||||
OptimizeTranscodingPayload,
|
||||
VideoTranscodingPayload
|
||||
} from '@shared/models'
|
||||
import { retryTransactionWrapper } from '../../../helpers/database-utils'
|
||||
import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg'
|
||||
import { logger, loggerTagsFactory } from '../../../helpers/logger'
|
||||
import { CONFIG } from '../../../initializers/config'
|
||||
import { VideoModel } from '../../../models/video/video'
|
||||
import {
|
||||
generateHlsPlaylistResolution,
|
||||
mergeAudioVideofile,
|
||||
optimizeOriginalVideofile,
|
||||
transcodeNewWebTorrentResolution
|
||||
} from '../../transcoding/transcoding'
|
||||
import { JobQueue } from '../job-queue'
|
||||
|
||||
type HandlerFunction = (job: Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<void>
|
||||
|
||||
@ -84,7 +74,37 @@ export {
|
||||
// Job handlers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, video: MVideoFullLight, user: MUser) {
|
||||
async function handleWebTorrentMergeAudioJob (job: Job, payload: MergeAudioTranscodingPayload, video: MVideoFullLight, user: MUserId) {
|
||||
logger.info('Handling merge audio transcoding job for %s.', video.uuid, lTags(video.uuid))
|
||||
|
||||
await mergeAudioVideofile({ video, resolution: payload.resolution, fps: payload.fps, job })
|
||||
|
||||
logger.info('Merge audio transcoding job for %s ended.', video.uuid, lTags(video.uuid))
|
||||
|
||||
await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video })
|
||||
}
|
||||
|
||||
async function handleWebTorrentOptimizeJob (job: Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) {
|
||||
logger.info('Handling optimize transcoding job for %s.', video.uuid, lTags(video.uuid))
|
||||
|
||||
await optimizeOriginalVideofile({ video, inputVideoFile: video.getMaxQualityFile(), quickTranscode: payload.quickTranscode, job })
|
||||
|
||||
logger.info('Optimize transcoding job for %s ended.', video.uuid, lTags(video.uuid))
|
||||
|
||||
await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video })
|
||||
}
|
||||
|
||||
async function handleNewWebTorrentResolutionJob (job: Job, payload: NewWebTorrentResolutionTranscodingPayload, video: MVideoFullLight) {
|
||||
logger.info('Handling WebTorrent transcoding job for %s.', video.uuid, lTags(video.uuid))
|
||||
|
||||
await transcodeNewWebTorrentResolution({ video, resolution: payload.resolution, fps: payload.fps, job })
|
||||
|
||||
logger.info('WebTorrent transcoding job for %s ended.', video.uuid, lTags(video.uuid))
|
||||
|
||||
await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video })
|
||||
}
|
||||
|
||||
async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, video: MVideoFullLight) {
|
||||
logger.info('Handling HLS transcoding job for %s.', video.uuid, lTags(video.uuid))
|
||||
|
||||
const videoFileInput = payload.copyCodecs
|
||||
@ -104,6 +124,7 @@ async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, video: MV
|
||||
videoInputPath,
|
||||
inputFileMutexReleaser,
|
||||
resolution: payload.resolution,
|
||||
fps: payload.fps,
|
||||
copyCodecs: payload.copyCodecs,
|
||||
job
|
||||
})
|
||||
@ -114,230 +135,11 @@ async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, video: MV
|
||||
|
||||
logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid))
|
||||
|
||||
await onHlsPlaylistGeneration(video, user, payload)
|
||||
}
|
||||
if (payload.deleteWebTorrentFiles === true) {
|
||||
logger.info('Removing WebTorrent files of %s now we have a HLS version of it.', video.uuid, lTags(video.uuid))
|
||||
|
||||
async function handleNewWebTorrentResolutionJob (
|
||||
job: Job,
|
||||
payload: NewWebTorrentResolutionTranscodingPayload,
|
||||
video: MVideoFullLight,
|
||||
user: MUserId
|
||||
) {
|
||||
logger.info('Handling WebTorrent transcoding job for %s.', video.uuid, lTags(video.uuid))
|
||||
|
||||
await transcodeNewWebTorrentResolution({ video, resolution: payload.resolution, job })
|
||||
|
||||
logger.info('WebTorrent transcoding job for %s ended.', video.uuid, lTags(video.uuid))
|
||||
|
||||
await onNewWebTorrentFileResolution(video, user, payload)
|
||||
}
|
||||
|
||||
async function handleWebTorrentMergeAudioJob (job: Job, payload: MergeAudioTranscodingPayload, video: MVideoFullLight, user: MUserId) {
|
||||
logger.info('Handling merge audio transcoding job for %s.', video.uuid, lTags(video.uuid))
|
||||
|
||||
await mergeAudioVideofile({ video, resolution: payload.resolution, job })
|
||||
|
||||
logger.info('Merge audio transcoding job for %s ended.', video.uuid, lTags(video.uuid))
|
||||
|
||||
await onVideoFirstWebTorrentTranscoding(video, payload, 'video', user)
|
||||
}
|
||||
|
||||
async function handleWebTorrentOptimizeJob (job: Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) {
|
||||
logger.info('Handling optimize transcoding job for %s.', video.uuid, lTags(video.uuid))
|
||||
|
||||
const { transcodeType } = await optimizeOriginalVideofile({ video, inputVideoFile: video.getMaxQualityFile(), job })
|
||||
|
||||
logger.info('Optimize transcoding job for %s ended.', video.uuid, lTags(video.uuid))
|
||||
|
||||
await onVideoFirstWebTorrentTranscoding(video, payload, transcodeType, user)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, payload: HLSTranscodingPayload) {
|
||||
if (payload.isMaxQuality && payload.autoDeleteWebTorrentIfNeeded && CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) {
|
||||
// Remove webtorrent files if not enabled
|
||||
for (const file of video.VideoFiles) {
|
||||
await video.removeWebTorrentFile(file)
|
||||
await file.destroy()
|
||||
}
|
||||
|
||||
video.VideoFiles = []
|
||||
|
||||
// Create HLS new resolution jobs
|
||||
await createLowerResolutionsJobs({
|
||||
video,
|
||||
user,
|
||||
videoFileResolution: payload.resolution,
|
||||
hasAudio: payload.hasAudio,
|
||||
isNewVideo: payload.isNewVideo ?? true,
|
||||
type: 'hls'
|
||||
})
|
||||
await removeAllWebTorrentFiles(video)
|
||||
}
|
||||
|
||||
await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
|
||||
await retryTransactionWrapper(moveToNextState, { video, isNewVideo: payload.isNewVideo })
|
||||
}
|
||||
|
||||
async function onVideoFirstWebTorrentTranscoding (
|
||||
videoArg: MVideoWithFile,
|
||||
payload: OptimizeTranscodingPayload | MergeAudioTranscodingPayload,
|
||||
transcodeType: TranscodeVODOptionsType,
|
||||
user: MUserId
|
||||
) {
|
||||
const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid)
|
||||
|
||||
try {
|
||||
// Maybe the video changed in database, refresh it
|
||||
const videoDatabase = await VideoModel.loadFull(videoArg.uuid)
|
||||
// Video does not exist anymore
|
||||
if (!videoDatabase) return undefined
|
||||
|
||||
const { resolution, audioStream } = await videoDatabase.probeMaxQualityFile()
|
||||
|
||||
// Generate HLS version of the original file
|
||||
const originalFileHLSPayload = {
|
||||
...payload,
|
||||
|
||||
hasAudio: !!audioStream,
|
||||
resolution: videoDatabase.getMaxQualityFile().resolution,
|
||||
// If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues
|
||||
copyCodecs: transcodeType !== 'quick-transcode',
|
||||
isMaxQuality: true
|
||||
}
|
||||
const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload)
|
||||
const hasNewResolutions = await createLowerResolutionsJobs({
|
||||
video: videoDatabase,
|
||||
user,
|
||||
videoFileResolution: resolution,
|
||||
hasAudio: !!audioStream,
|
||||
type: 'webtorrent',
|
||||
isNewVideo: payload.isNewVideo ?? true
|
||||
})
|
||||
|
||||
await VideoJobInfoModel.decrease(videoDatabase.uuid, 'pendingTranscode')
|
||||
|
||||
// Move to next state if there are no other resolutions to generate
|
||||
if (!hasHls && !hasNewResolutions) {
|
||||
await retryTransactionWrapper(moveToNextState, { video: videoDatabase, isNewVideo: payload.isNewVideo })
|
||||
}
|
||||
} finally {
|
||||
mutexReleaser()
|
||||
}
|
||||
}
|
||||
|
||||
async function onNewWebTorrentFileResolution (
|
||||
video: MVideo,
|
||||
user: MUserId,
|
||||
payload: NewWebTorrentResolutionTranscodingPayload | MergeAudioTranscodingPayload
|
||||
) {
|
||||
if (payload.createHLSIfNeeded) {
|
||||
await createHlsJobIfEnabled(user, { hasAudio: true, copyCodecs: true, isMaxQuality: false, ...payload })
|
||||
}
|
||||
|
||||
await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
|
||||
|
||||
await retryTransactionWrapper(moveToNextState, { video, isNewVideo: payload.isNewVideo })
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function createHlsJobIfEnabled (user: MUserId, payload: {
|
||||
videoUUID: string
|
||||
resolution: number
|
||||
hasAudio: boolean
|
||||
copyCodecs: boolean
|
||||
isMaxQuality: boolean
|
||||
isNewVideo?: boolean
|
||||
}) {
|
||||
if (!payload || CONFIG.TRANSCODING.ENABLED !== true || CONFIG.TRANSCODING.HLS.ENABLED !== true) return false
|
||||
|
||||
const jobOptions = {
|
||||
priority: await getTranscodingJobPriority(user)
|
||||
}
|
||||
|
||||
const hlsTranscodingPayload: HLSTranscodingPayload = {
|
||||
type: 'new-resolution-to-hls',
|
||||
autoDeleteWebTorrentIfNeeded: true,
|
||||
|
||||
...pick(payload, [ 'videoUUID', 'resolution', 'copyCodecs', 'isMaxQuality', 'isNewVideo', 'hasAudio' ])
|
||||
}
|
||||
|
||||
await JobQueue.Instance.createJob(await buildTranscodingJob(hlsTranscodingPayload, jobOptions))
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
async function createLowerResolutionsJobs (options: {
|
||||
video: MVideoFullLight
|
||||
user: MUserId
|
||||
videoFileResolution: number
|
||||
hasAudio: boolean
|
||||
isNewVideo: boolean
|
||||
type: 'hls' | 'webtorrent'
|
||||
}) {
|
||||
const { video, user, videoFileResolution, isNewVideo, hasAudio, type } = options
|
||||
|
||||
// Create transcoding jobs if there are enabled resolutions
|
||||
const resolutionsEnabled = await Hooks.wrapObject(
|
||||
computeResolutionsToTranscode({ input: videoFileResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }),
|
||||
'filter:transcoding.auto.resolutions-to-transcode.result',
|
||||
options
|
||||
)
|
||||
|
||||
const resolutionCreated: string[] = []
|
||||
|
||||
for (const resolution of resolutionsEnabled) {
|
||||
let dataInput: VideoTranscodingPayload
|
||||
|
||||
if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED && type === 'webtorrent') {
|
||||
// WebTorrent will create subsequent HLS job
|
||||
dataInput = {
|
||||
type: 'new-resolution-to-webtorrent',
|
||||
videoUUID: video.uuid,
|
||||
resolution,
|
||||
hasAudio,
|
||||
createHLSIfNeeded: true,
|
||||
isNewVideo
|
||||
}
|
||||
|
||||
resolutionCreated.push('webtorrent-' + resolution)
|
||||
}
|
||||
|
||||
if (CONFIG.TRANSCODING.HLS.ENABLED && type === 'hls') {
|
||||
dataInput = {
|
||||
type: 'new-resolution-to-hls',
|
||||
videoUUID: video.uuid,
|
||||
resolution,
|
||||
hasAudio,
|
||||
copyCodecs: false,
|
||||
isMaxQuality: false,
|
||||
autoDeleteWebTorrentIfNeeded: true,
|
||||
isNewVideo
|
||||
}
|
||||
|
||||
resolutionCreated.push('hls-' + resolution)
|
||||
}
|
||||
|
||||
if (!dataInput) continue
|
||||
|
||||
const jobOptions = {
|
||||
priority: await getTranscodingJobPriority(user)
|
||||
}
|
||||
|
||||
await JobQueue.Instance.createJob(await buildTranscodingJob(dataInput, jobOptions))
|
||||
}
|
||||
|
||||
if (resolutionCreated.length === 0) {
|
||||
logger.info('No transcoding jobs created for video %s (no resolutions).', video.uuid, lTags(video.uuid))
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
logger.info(
|
||||
'New resolutions %s transcoding jobs created for video %s and origin file resolution of %d.', type, video.uuid, videoFileResolution,
|
||||
{ resolutionCreated, ...lTags(video.uuid) }
|
||||
)
|
||||
|
||||
return true
|
||||
await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video })
|
||||
}
|
||||
|
@ -31,6 +31,7 @@ import {
|
||||
MoveObjectStoragePayload,
|
||||
NotifyPayload,
|
||||
RefreshPayload,
|
||||
TranscodingJobBuilderPayload,
|
||||
VideoChannelImportPayload,
|
||||
VideoFileImportPayload,
|
||||
VideoImportPayload,
|
||||
@ -56,6 +57,7 @@ import { processFederateVideo } from './handlers/federate-video'
|
||||
import { processManageVideoTorrent } from './handlers/manage-video-torrent'
|
||||
import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage'
|
||||
import { processNotify } from './handlers/notify'
|
||||
import { processTranscodingJobBuilder } from './handlers/transcoding-job-builder'
|
||||
import { processVideoChannelImport } from './handlers/video-channel-import'
|
||||
import { processVideoFileImport } from './handlers/video-file-import'
|
||||
import { processVideoImport } from './handlers/video-import'
|
||||
@ -69,11 +71,12 @@ export type CreateJobArgument =
|
||||
{ type: 'activitypub-http-broadcast-parallel', payload: ActivitypubHttpBroadcastPayload } |
|
||||
{ type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } |
|
||||
{ type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } |
|
||||
{ type: 'activitypub-http-cleaner', payload: {} } |
|
||||
{ type: 'activitypub-cleaner', payload: {} } |
|
||||
{ type: 'activitypub-follow', payload: ActivitypubFollowPayload } |
|
||||
{ type: 'video-file-import', payload: VideoFileImportPayload } |
|
||||
{ type: 'video-transcoding', payload: VideoTranscodingPayload } |
|
||||
{ type: 'email', payload: EmailPayload } |
|
||||
{ type: 'transcoding-job-builder', payload: TranscodingJobBuilderPayload } |
|
||||
{ type: 'video-import', payload: VideoImportPayload } |
|
||||
{ type: 'activitypub-refresher', payload: RefreshPayload } |
|
||||
{ type: 'videos-views-stats', payload: {} } |
|
||||
@ -96,28 +99,29 @@ export type CreateJobOptions = {
|
||||
}
|
||||
|
||||
const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {
|
||||
'activitypub-http-broadcast': processActivityPubHttpSequentialBroadcast,
|
||||
'activitypub-http-broadcast-parallel': processActivityPubParallelHttpBroadcast,
|
||||
'activitypub-http-unicast': processActivityPubHttpUnicast,
|
||||
'activitypub-http-fetcher': processActivityPubHttpFetcher,
|
||||
'activitypub-cleaner': processActivityPubCleaner,
|
||||
'activitypub-follow': processActivityPubFollow,
|
||||
'video-file-import': processVideoFileImport,
|
||||
'video-transcoding': processVideoTranscoding,
|
||||
'email': processEmail,
|
||||
'video-import': processVideoImport,
|
||||
'videos-views-stats': processVideosViewsStats,
|
||||
'activitypub-http-broadcast-parallel': processActivityPubParallelHttpBroadcast,
|
||||
'activitypub-http-broadcast': processActivityPubHttpSequentialBroadcast,
|
||||
'activitypub-http-fetcher': processActivityPubHttpFetcher,
|
||||
'activitypub-http-unicast': processActivityPubHttpUnicast,
|
||||
'activitypub-refresher': refreshAPObject,
|
||||
'video-live-ending': processVideoLiveEnding,
|
||||
'actor-keys': processActorKeys,
|
||||
'video-redundancy': processVideoRedundancy,
|
||||
'move-to-object-storage': processMoveToObjectStorage,
|
||||
'manage-video-torrent': processManageVideoTorrent,
|
||||
'video-studio-edition': processVideoStudioEdition,
|
||||
'video-channel-import': processVideoChannelImport,
|
||||
'after-video-channel-import': processAfterVideoChannelImport,
|
||||
'email': processEmail,
|
||||
'federate-video': processFederateVideo,
|
||||
'transcoding-job-builder': processTranscodingJobBuilder,
|
||||
'manage-video-torrent': processManageVideoTorrent,
|
||||
'move-to-object-storage': processMoveToObjectStorage,
|
||||
'notify': processNotify,
|
||||
'federate-video': processFederateVideo
|
||||
'video-channel-import': processVideoChannelImport,
|
||||
'video-file-import': processVideoFileImport,
|
||||
'video-import': processVideoImport,
|
||||
'video-live-ending': processVideoLiveEnding,
|
||||
'video-redundancy': processVideoRedundancy,
|
||||
'video-studio-edition': processVideoStudioEdition,
|
||||
'video-transcoding': processVideoTranscoding,
|
||||
'videos-views-stats': processVideosViewsStats
|
||||
}
|
||||
|
||||
const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } = {
|
||||
@ -125,28 +129,29 @@ const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> }
|
||||
}
|
||||
|
||||
const jobTypes: JobType[] = [
|
||||
'activitypub-cleaner',
|
||||
'activitypub-follow',
|
||||
'activitypub-http-broadcast',
|
||||
'activitypub-http-broadcast-parallel',
|
||||
'activitypub-http-broadcast',
|
||||
'activitypub-http-fetcher',
|
||||
'activitypub-http-unicast',
|
||||
'activitypub-cleaner',
|
||||
'activitypub-refresher',
|
||||
'actor-keys',
|
||||
'after-video-channel-import',
|
||||
'email',
|
||||
'video-transcoding',
|
||||
'federate-video',
|
||||
'transcoding-job-builder',
|
||||
'manage-video-torrent',
|
||||
'move-to-object-storage',
|
||||
'notify',
|
||||
'video-channel-import',
|
||||
'video-file-import',
|
||||
'video-import',
|
||||
'videos-views-stats',
|
||||
'activitypub-refresher',
|
||||
'video-redundancy',
|
||||
'actor-keys',
|
||||
'video-live-ending',
|
||||
'move-to-object-storage',
|
||||
'manage-video-torrent',
|
||||
'video-redundancy',
|
||||
'video-studio-edition',
|
||||
'video-channel-import',
|
||||
'after-video-channel-import',
|
||||
'notify',
|
||||
'federate-video'
|
||||
'video-transcoding',
|
||||
'videos-views-stats'
|
||||
]
|
||||
|
||||
const silentFailure = new Set<JobType>([ 'activitypub-http-unicast' ])
|
||||
|
@ -2,36 +2,30 @@ import { readdir, readFile } from 'fs-extra'
|
||||
import { createServer, Server } from 'net'
|
||||
import { join } from 'path'
|
||||
import { createServer as createServerTLS, Server as ServerTLS } from 'tls'
|
||||
import {
|
||||
computeResolutionsToTranscode,
|
||||
ffprobePromise,
|
||||
getLiveSegmentTime,
|
||||
getVideoStreamBitrate,
|
||||
getVideoStreamDimensionsInfo,
|
||||
getVideoStreamFPS,
|
||||
hasAudioStream
|
||||
} from '@server/helpers/ffmpeg'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger'
|
||||
import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config'
|
||||
import { VIDEO_LIVE } from '@server/initializers/constants'
|
||||
import { sequelizeTypescript } from '@server/initializers/database'
|
||||
import { UserModel } from '@server/models/user/user'
|
||||
import { VideoModel } from '@server/models/video/video'
|
||||
import { VideoLiveModel } from '@server/models/video/video-live'
|
||||
import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting'
|
||||
import { VideoLiveSessionModel } from '@server/models/video/video-live-session'
|
||||
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
|
||||
import { MVideo, MVideoLiveSession, MVideoLiveVideo, MVideoLiveVideoWithSetting } from '@server/types/models'
|
||||
import { pick, wait } from '@shared/core-utils'
|
||||
import { ffprobePromise, getVideoStreamBitrate, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream } from '@shared/ffmpeg'
|
||||
import { LiveVideoError, VideoState } from '@shared/models'
|
||||
import { federateVideoIfNeeded } from '../activitypub/videos'
|
||||
import { JobQueue } from '../job-queue'
|
||||
import { getLiveReplayBaseDirectory } from '../paths'
|
||||
import { PeerTubeSocket } from '../peertube-socket'
|
||||
import { Hooks } from '../plugins/hooks'
|
||||
import { computeResolutionsToTranscode } from '../transcoding/transcoding-resolutions'
|
||||
import { LiveQuotaStore } from './live-quota-store'
|
||||
import { cleanupAndDestroyPermanentLive } from './live-utils'
|
||||
import { cleanupAndDestroyPermanentLive, getLiveSegmentTime } from './live-utils'
|
||||
import { MuxingSession } from './shared'
|
||||
import { sequelizeTypescript } from '@server/initializers/database'
|
||||
import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting'
|
||||
import { RunnerJobModel } from '@server/models/runner/runner-job'
|
||||
|
||||
const NodeRtmpSession = require('node-media-server/src/node_rtmp_session')
|
||||
const context = require('node-media-server/src/node_core_ctx')
|
||||
@ -57,7 +51,7 @@ class LiveManager {
|
||||
private static instance: LiveManager
|
||||
|
||||
private readonly muxingSessions = new Map<string, MuxingSession>()
|
||||
private readonly videoSessions = new Map<number, string>()
|
||||
private readonly videoSessions = new Map<string, string>()
|
||||
|
||||
private rtmpServer: Server
|
||||
private rtmpsServer: ServerTLS
|
||||
@ -177,14 +171,19 @@ class LiveManager {
|
||||
return !!this.rtmpServer
|
||||
}
|
||||
|
||||
stopSessionOf (videoId: number, error: LiveVideoError | null) {
|
||||
const sessionId = this.videoSessions.get(videoId)
|
||||
if (!sessionId) return
|
||||
stopSessionOf (videoUUID: string, error: LiveVideoError | null) {
|
||||
const sessionId = this.videoSessions.get(videoUUID)
|
||||
if (!sessionId) {
|
||||
logger.debug('No live session to stop for video %s', videoUUID, lTags(sessionId, videoUUID))
|
||||
return
|
||||
}
|
||||
|
||||
this.saveEndingSession(videoId, error)
|
||||
.catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId) }))
|
||||
logger.info('Stopping live session of video %s', videoUUID, { error, ...lTags(sessionId, videoUUID) })
|
||||
|
||||
this.videoSessions.delete(videoId)
|
||||
this.saveEndingSession(videoUUID, error)
|
||||
.catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId, videoUUID) }))
|
||||
|
||||
this.videoSessions.delete(videoUUID)
|
||||
this.abortSession(sessionId)
|
||||
}
|
||||
|
||||
@ -221,6 +220,11 @@ class LiveManager {
|
||||
return this.abortSession(sessionId)
|
||||
}
|
||||
|
||||
if (this.videoSessions.has(video.uuid)) {
|
||||
logger.warn('Video %s has already a live session. Refusing stream %s.', video.uuid, streamKey, lTags(sessionId, video.uuid))
|
||||
return this.abortSession(sessionId)
|
||||
}
|
||||
|
||||
// Cleanup old potential live (could happen with a permanent live)
|
||||
const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id)
|
||||
if (oldStreamingPlaylist) {
|
||||
@ -229,7 +233,7 @@ class LiveManager {
|
||||
await cleanupAndDestroyPermanentLive(video, oldStreamingPlaylist)
|
||||
}
|
||||
|
||||
this.videoSessions.set(video.id, sessionId)
|
||||
this.videoSessions.set(video.uuid, sessionId)
|
||||
|
||||
const now = Date.now()
|
||||
const probe = await ffprobePromise(inputUrl)
|
||||
@ -253,7 +257,7 @@ class LiveManager {
|
||||
)
|
||||
|
||||
logger.info(
|
||||
'Will mux/transcode live video of original resolution %d.', resolution,
|
||||
'Handling live video of original resolution %d.', resolution,
|
||||
{ allResolutions, ...lTags(sessionId, video.uuid) }
|
||||
)
|
||||
|
||||
@ -301,44 +305,44 @@ class LiveManager {
|
||||
|
||||
muxingSession.on('live-ready', () => this.publishAndFederateLive(videoLive, localLTags))
|
||||
|
||||
muxingSession.on('bad-socket-health', ({ videoId }) => {
|
||||
muxingSession.on('bad-socket-health', ({ videoUUID }) => {
|
||||
logger.error(
|
||||
'Too much data in client socket stream (ffmpeg is too slow to transcode the video).' +
|
||||
' Stopping session of video %s.', videoUUID,
|
||||
localLTags
|
||||
)
|
||||
|
||||
this.stopSessionOf(videoId, LiveVideoError.BAD_SOCKET_HEALTH)
|
||||
this.stopSessionOf(videoUUID, LiveVideoError.BAD_SOCKET_HEALTH)
|
||||
})
|
||||
|
||||
muxingSession.on('duration-exceeded', ({ videoId }) => {
|
||||
muxingSession.on('duration-exceeded', ({ videoUUID }) => {
|
||||
logger.info('Stopping session of %s: max duration exceeded.', videoUUID, localLTags)
|
||||
|
||||
this.stopSessionOf(videoId, LiveVideoError.DURATION_EXCEEDED)
|
||||
this.stopSessionOf(videoUUID, LiveVideoError.DURATION_EXCEEDED)
|
||||
})
|
||||
|
||||
muxingSession.on('quota-exceeded', ({ videoId }) => {
|
||||
muxingSession.on('quota-exceeded', ({ videoUUID }) => {
|
||||
logger.info('Stopping session of %s: user quota exceeded.', videoUUID, localLTags)
|
||||
|
||||
this.stopSessionOf(videoId, LiveVideoError.QUOTA_EXCEEDED)
|
||||
this.stopSessionOf(videoUUID, LiveVideoError.QUOTA_EXCEEDED)
|
||||
})
|
||||
|
||||
muxingSession.on('ffmpeg-error', ({ videoId }) => {
|
||||
this.stopSessionOf(videoId, LiveVideoError.FFMPEG_ERROR)
|
||||
muxingSession.on('transcoding-error', ({ videoUUID }) => {
|
||||
this.stopSessionOf(videoUUID, LiveVideoError.FFMPEG_ERROR)
|
||||
})
|
||||
|
||||
muxingSession.on('ffmpeg-end', ({ videoId }) => {
|
||||
this.onMuxingFFmpegEnd(videoId, sessionId)
|
||||
muxingSession.on('transcoding-end', ({ videoUUID }) => {
|
||||
this.onMuxingFFmpegEnd(videoUUID, sessionId)
|
||||
})
|
||||
|
||||
muxingSession.on('after-cleanup', ({ videoId }) => {
|
||||
muxingSession.on('after-cleanup', ({ videoUUID }) => {
|
||||
this.muxingSessions.delete(sessionId)
|
||||
|
||||
LiveQuotaStore.Instance.removeLive(user.id, videoLive.id)
|
||||
|
||||
muxingSession.destroy()
|
||||
|
||||
return this.onAfterMuxingCleanup({ videoId, liveSession })
|
||||
return this.onAfterMuxingCleanup({ videoUUID, liveSession })
|
||||
.catch(err => logger.error('Error in end transmuxing.', { err, ...localLTags }))
|
||||
})
|
||||
|
||||
@ -379,22 +383,24 @@ class LiveManager {
|
||||
}
|
||||
}
|
||||
|
||||
private onMuxingFFmpegEnd (videoId: number, sessionId: string) {
|
||||
this.videoSessions.delete(videoId)
|
||||
private onMuxingFFmpegEnd (videoUUID: string, sessionId: string) {
|
||||
this.videoSessions.delete(videoUUID)
|
||||
|
||||
this.saveEndingSession(videoId, null)
|
||||
this.saveEndingSession(videoUUID, null)
|
||||
.catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId) }))
|
||||
}
|
||||
|
||||
private async onAfterMuxingCleanup (options: {
|
||||
videoId: number | string
|
||||
videoUUID: string
|
||||
liveSession?: MVideoLiveSession
|
||||
cleanupNow?: boolean // Default false
|
||||
}) {
|
||||
const { videoId, liveSession: liveSessionArg, cleanupNow = false } = options
|
||||
const { videoUUID, liveSession: liveSessionArg, cleanupNow = false } = options
|
||||
|
||||
logger.debug('Live of video %s has been cleaned up. Moving to its next state.', videoUUID, lTags(videoUUID))
|
||||
|
||||
try {
|
||||
const fullVideo = await VideoModel.loadFull(videoId)
|
||||
const fullVideo = await VideoModel.loadFull(videoUUID)
|
||||
if (!fullVideo) return
|
||||
|
||||
const live = await VideoLiveModel.loadByVideoId(fullVideo.id)
|
||||
@ -437,15 +443,17 @@ class LiveManager {
|
||||
|
||||
await federateVideoIfNeeded(fullVideo, false)
|
||||
} catch (err) {
|
||||
logger.error('Cannot save/federate new video state of live streaming of video %d.', videoId, { err, ...lTags(videoId + '') })
|
||||
logger.error('Cannot save/federate new video state of live streaming of video %s.', videoUUID, { err, ...lTags(videoUUID) })
|
||||
}
|
||||
}
|
||||
|
||||
private async handleBrokenLives () {
|
||||
await RunnerJobModel.cancelAllJobs({ type: 'live-rtmp-hls-transcoding' })
|
||||
|
||||
const videoUUIDs = await VideoModel.listPublishedLiveUUIDs()
|
||||
|
||||
for (const uuid of videoUUIDs) {
|
||||
await this.onAfterMuxingCleanup({ videoId: uuid, cleanupNow: true })
|
||||
await this.onAfterMuxingCleanup({ videoUUID: uuid, cleanupNow: true })
|
||||
}
|
||||
}
|
||||
|
||||
@ -494,8 +502,8 @@ class LiveManager {
|
||||
})
|
||||
}
|
||||
|
||||
private async saveEndingSession (videoId: number, error: LiveVideoError | null) {
|
||||
const liveSession = await VideoLiveSessionModel.findCurrentSessionOf(videoId)
|
||||
private async saveEndingSession (videoUUID: string, error: LiveVideoError | null) {
|
||||
const liveSession = await VideoLiveSessionModel.findCurrentSessionOf(videoUUID)
|
||||
if (!liveSession) return
|
||||
|
||||
liveSession.endDate = new Date()
|
||||
|
@ -52,7 +52,10 @@ class LiveSegmentShaStore {
|
||||
logger.debug('Removing live sha segment %s.', segmentPath, lTags(this.videoUUID))
|
||||
|
||||
if (!this.segmentsSha256.has(segmentName)) {
|
||||
logger.warn('Unknown segment in files map for video %s and segment %s.', this.videoUUID, segmentPath, lTags(this.videoUUID))
|
||||
logger.warn(
|
||||
'Unknown segment in live segment hash store for video %s and segment %s.',
|
||||
this.videoUUID, segmentPath, lTags(this.videoUUID)
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -1,8 +1,9 @@
|
||||
import { pathExists, readdir, remove } from 'fs-extra'
|
||||
import { basename, join } from 'path'
|
||||
import { logger } from '@server/helpers/logger'
|
||||
import { VIDEO_LIVE } from '@server/initializers/constants'
|
||||
import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models'
|
||||
import { VideoStorage } from '@shared/models'
|
||||
import { LiveVideoLatencyMode, VideoStorage } from '@shared/models'
|
||||
import { listHLSFileKeysOf, removeHLSFileObjectStorageByFullKey, removeHLSObjectStorage } from '../object-storage'
|
||||
import { getLiveDirectory } from '../paths'
|
||||
|
||||
@ -37,10 +38,19 @@ async function cleanupTMPLiveFiles (video: MVideo, streamingPlaylist: MStreaming
|
||||
await cleanupTMPLiveFilesFromFilesystem(video)
|
||||
}
|
||||
|
||||
function getLiveSegmentTime (latencyMode: LiveVideoLatencyMode) {
|
||||
if (latencyMode === LiveVideoLatencyMode.SMALL_LATENCY) {
|
||||
return VIDEO_LIVE.SEGMENT_TIME_SECONDS.SMALL_LATENCY
|
||||
}
|
||||
|
||||
return VIDEO_LIVE.SEGMENT_TIME_SECONDS.DEFAULT_LATENCY
|
||||
}
|
||||
|
||||
export {
|
||||
cleanupAndDestroyPermanentLive,
|
||||
cleanupUnsavedNormalLive,
|
||||
cleanupTMPLiveFiles,
|
||||
getLiveSegmentTime,
|
||||
buildConcatenatedName
|
||||
}
|
||||
|
||||
|
@ -1,11 +1,10 @@
|
||||
import { mapSeries } from 'bluebird'
|
||||
import { FSWatcher, watch } from 'chokidar'
|
||||
import { FfmpegCommand } from 'fluent-ffmpeg'
|
||||
import { EventEmitter } from 'events'
|
||||
import { appendFile, ensureDir, readFile, stat } from 'fs-extra'
|
||||
import PQueue from 'p-queue'
|
||||
import { basename, join } from 'path'
|
||||
import { EventEmitter } from 'stream'
|
||||
import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers/ffmpeg'
|
||||
import { computeOutputFPS } from '@server/helpers/ffmpeg'
|
||||
import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger'
|
||||
import { CONFIG } from '@server/initializers/config'
|
||||
import { MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE } from '@server/initializers/constants'
|
||||
@ -20,24 +19,24 @@ import {
|
||||
getLiveDirectory,
|
||||
getLiveReplayBaseDirectory
|
||||
} from '../../paths'
|
||||
import { VideoTranscodingProfilesManager } from '../../transcoding/default-transcoding-profiles'
|
||||
import { isAbleToUploadVideo } from '../../user'
|
||||
import { LiveQuotaStore } from '../live-quota-store'
|
||||
import { LiveSegmentShaStore } from '../live-segment-sha-store'
|
||||
import { buildConcatenatedName } from '../live-utils'
|
||||
import { buildConcatenatedName, getLiveSegmentTime } from '../live-utils'
|
||||
import { AbstractTranscodingWrapper, FFmpegTranscodingWrapper, RemoteTranscodingWrapper } from './transcoding-wrapper'
|
||||
|
||||
import memoizee = require('memoizee')
|
||||
interface MuxingSessionEvents {
|
||||
'live-ready': (options: { videoId: number }) => void
|
||||
'live-ready': (options: { videoUUID: string }) => void
|
||||
|
||||
'bad-socket-health': (options: { videoId: number }) => void
|
||||
'duration-exceeded': (options: { videoId: number }) => void
|
||||
'quota-exceeded': (options: { videoId: number }) => void
|
||||
'bad-socket-health': (options: { videoUUID: string }) => void
|
||||
'duration-exceeded': (options: { videoUUID: string }) => void
|
||||
'quota-exceeded': (options: { videoUUID: string }) => void
|
||||
|
||||
'ffmpeg-end': (options: { videoId: number }) => void
|
||||
'ffmpeg-error': (options: { videoId: number }) => void
|
||||
'transcoding-end': (options: { videoUUID: string }) => void
|
||||
'transcoding-error': (options: { videoUUID: string }) => void
|
||||
|
||||
'after-cleanup': (options: { videoId: number }) => void
|
||||
'after-cleanup': (options: { videoUUID: string }) => void
|
||||
}
|
||||
|
||||
declare interface MuxingSession {
|
||||
@ -52,7 +51,7 @@ declare interface MuxingSession {
|
||||
|
||||
class MuxingSession extends EventEmitter {
|
||||
|
||||
private ffmpegCommand: FfmpegCommand
|
||||
private transcodingWrapper: AbstractTranscodingWrapper
|
||||
|
||||
private readonly context: any
|
||||
private readonly user: MUserId
|
||||
@ -67,7 +66,6 @@ class MuxingSession extends EventEmitter {
|
||||
|
||||
private readonly hasAudio: boolean
|
||||
|
||||
private readonly videoId: number
|
||||
private readonly videoUUID: string
|
||||
private readonly saveReplay: boolean
|
||||
|
||||
@ -126,7 +124,6 @@ class MuxingSession extends EventEmitter {
|
||||
|
||||
this.allResolutions = options.allResolutions
|
||||
|
||||
this.videoId = this.videoLive.Video.id
|
||||
this.videoUUID = this.videoLive.Video.uuid
|
||||
|
||||
this.saveReplay = this.videoLive.saveReplay
|
||||
@ -145,63 +142,23 @@ class MuxingSession extends EventEmitter {
|
||||
|
||||
await this.prepareDirectories()
|
||||
|
||||
this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED
|
||||
? await getLiveTranscodingCommand({
|
||||
inputUrl: this.inputUrl,
|
||||
this.transcodingWrapper = this.buildTranscodingWrapper()
|
||||
|
||||
outPath: this.outDirectory,
|
||||
masterPlaylistName: this.streamingPlaylist.playlistFilename,
|
||||
this.transcodingWrapper.on('end', () => this.onTranscodedEnded())
|
||||
this.transcodingWrapper.on('error', () => this.onTranscodingError())
|
||||
|
||||
latencyMode: this.videoLive.latencyMode,
|
||||
|
||||
resolutions: this.allResolutions,
|
||||
fps: this.fps,
|
||||
bitrate: this.bitrate,
|
||||
ratio: this.ratio,
|
||||
|
||||
hasAudio: this.hasAudio,
|
||||
|
||||
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
|
||||
profile: CONFIG.LIVE.TRANSCODING.PROFILE
|
||||
})
|
||||
: getLiveMuxingCommand({
|
||||
inputUrl: this.inputUrl,
|
||||
outPath: this.outDirectory,
|
||||
masterPlaylistName: this.streamingPlaylist.playlistFilename,
|
||||
latencyMode: this.videoLive.latencyMode
|
||||
})
|
||||
|
||||
logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags())
|
||||
await this.transcodingWrapper.run()
|
||||
|
||||
this.watchMasterFile()
|
||||
this.watchTSFiles()
|
||||
this.watchM3U8File()
|
||||
|
||||
let ffmpegShellCommand: string
|
||||
this.ffmpegCommand.on('start', cmdline => {
|
||||
ffmpegShellCommand = cmdline
|
||||
|
||||
logger.debug('Running ffmpeg command for live', { ffmpegShellCommand, ...this.lTags() })
|
||||
})
|
||||
|
||||
this.ffmpegCommand.on('error', (err, stdout, stderr) => {
|
||||
this.onFFmpegError({ err, stdout, stderr, ffmpegShellCommand })
|
||||
})
|
||||
|
||||
this.ffmpegCommand.on('end', () => {
|
||||
this.emit('ffmpeg-end', ({ videoId: this.videoId }))
|
||||
|
||||
this.onFFmpegEnded()
|
||||
})
|
||||
|
||||
this.ffmpegCommand.run()
|
||||
}
|
||||
|
||||
abort () {
|
||||
if (!this.ffmpegCommand) return
|
||||
if (!this.transcodingWrapper) return
|
||||
|
||||
this.aborted = true
|
||||
this.ffmpegCommand.kill('SIGINT')
|
||||
this.transcodingWrapper.abort()
|
||||
}
|
||||
|
||||
destroy () {
|
||||
@ -210,48 +167,6 @@ class MuxingSession extends EventEmitter {
|
||||
this.hasClientSocketInBadHealthWithCache.clear()
|
||||
}
|
||||
|
||||
private onFFmpegError (options: {
|
||||
err: any
|
||||
stdout: string
|
||||
stderr: string
|
||||
ffmpegShellCommand: string
|
||||
}) {
|
||||
const { err, stdout, stderr, ffmpegShellCommand } = options
|
||||
|
||||
this.onFFmpegEnded()
|
||||
|
||||
// Don't care that we killed the ffmpeg process
|
||||
if (err?.message?.includes('Exiting normally')) return
|
||||
|
||||
logger.error('Live transcoding error.', { err, stdout, stderr, ffmpegShellCommand, ...this.lTags() })
|
||||
|
||||
this.emit('ffmpeg-error', ({ videoId: this.videoId }))
|
||||
}
|
||||
|
||||
private onFFmpegEnded () {
|
||||
logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', this.inputUrl, this.lTags())
|
||||
|
||||
setTimeout(() => {
|
||||
// Wait latest segments generation, and close watchers
|
||||
|
||||
Promise.all([ this.tsWatcher.close(), this.masterWatcher.close(), this.m3u8Watcher.close() ])
|
||||
.then(() => {
|
||||
// Process remaining segments hash
|
||||
for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) {
|
||||
this.processSegments(this.segmentsToProcessPerPlaylist[key])
|
||||
}
|
||||
})
|
||||
.catch(err => {
|
||||
logger.error(
|
||||
'Cannot close watchers of %s or process remaining hash segments.', this.outDirectory,
|
||||
{ err, ...this.lTags() }
|
||||
)
|
||||
})
|
||||
|
||||
this.emit('after-cleanup', { videoId: this.videoId })
|
||||
}, 1000)
|
||||
}
|
||||
|
||||
private watchMasterFile () {
|
||||
this.masterWatcher = watch(this.outDirectory + '/' + this.streamingPlaylist.playlistFilename)
|
||||
|
||||
@ -272,6 +187,8 @@ class MuxingSession extends EventEmitter {
|
||||
|
||||
this.masterPlaylistCreated = true
|
||||
|
||||
logger.info('Master playlist file for %s has been created', this.videoUUID, this.lTags())
|
||||
|
||||
this.masterWatcher.close()
|
||||
.catch(err => logger.error('Cannot close master watcher of %s.', this.outDirectory, { err, ...this.lTags() }))
|
||||
})
|
||||
@ -318,19 +235,19 @@ class MuxingSession extends EventEmitter {
|
||||
this.segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ]
|
||||
|
||||
if (this.hasClientSocketInBadHealthWithCache(this.sessionId)) {
|
||||
this.emit('bad-socket-health', { videoId: this.videoId })
|
||||
this.emit('bad-socket-health', { videoUUID: this.videoUUID })
|
||||
return
|
||||
}
|
||||
|
||||
// Duration constraint check
|
||||
if (this.isDurationConstraintValid(startStreamDateTime) !== true) {
|
||||
this.emit('duration-exceeded', { videoId: this.videoId })
|
||||
this.emit('duration-exceeded', { videoUUID: this.videoUUID })
|
||||
return
|
||||
}
|
||||
|
||||
// Check user quota if the user enabled replay saving
|
||||
if (await this.isQuotaExceeded(segmentPath) === true) {
|
||||
this.emit('quota-exceeded', { videoId: this.videoId })
|
||||
this.emit('quota-exceeded', { videoUUID: this.videoUUID })
|
||||
}
|
||||
}
|
||||
|
||||
@ -438,10 +355,40 @@ class MuxingSession extends EventEmitter {
|
||||
if (this.masterPlaylistCreated && !this.liveReady) {
|
||||
this.liveReady = true
|
||||
|
||||
this.emit('live-ready', { videoId: this.videoId })
|
||||
this.emit('live-ready', { videoUUID: this.videoUUID })
|
||||
}
|
||||
}
|
||||
|
||||
private onTranscodingError () {
|
||||
this.emit('transcoding-error', ({ videoUUID: this.videoUUID }))
|
||||
}
|
||||
|
||||
private onTranscodedEnded () {
|
||||
this.emit('transcoding-end', ({ videoUUID: this.videoUUID }))
|
||||
|
||||
logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', this.inputUrl, this.lTags())
|
||||
|
||||
setTimeout(() => {
|
||||
// Wait latest segments generation, and close watchers
|
||||
|
||||
Promise.all([ this.tsWatcher.close(), this.masterWatcher.close(), this.m3u8Watcher.close() ])
|
||||
.then(() => {
|
||||
// Process remaining segments hash
|
||||
for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) {
|
||||
this.processSegments(this.segmentsToProcessPerPlaylist[key])
|
||||
}
|
||||
})
|
||||
.catch(err => {
|
||||
logger.error(
|
||||
'Cannot close watchers of %s or process remaining hash segments.', this.outDirectory,
|
||||
{ err, ...this.lTags() }
|
||||
)
|
||||
})
|
||||
|
||||
this.emit('after-cleanup', { videoUUID: this.videoUUID })
|
||||
}, 1000)
|
||||
}
|
||||
|
||||
private hasClientSocketInBadHealth (sessionId: string) {
|
||||
const rtmpSession = this.context.sessions.get(sessionId)
|
||||
|
||||
@ -503,6 +450,36 @@ class MuxingSession extends EventEmitter {
|
||||
sendToObjectStorage: CONFIG.OBJECT_STORAGE.ENABLED
|
||||
})
|
||||
}
|
||||
|
||||
private buildTranscodingWrapper () {
|
||||
const options = {
|
||||
streamingPlaylist: this.streamingPlaylist,
|
||||
videoLive: this.videoLive,
|
||||
|
||||
lTags: this.lTags,
|
||||
|
||||
inputUrl: this.inputUrl,
|
||||
|
||||
toTranscode: this.allResolutions.map(resolution => ({
|
||||
resolution,
|
||||
fps: computeOutputFPS({ inputFPS: this.fps, resolution })
|
||||
})),
|
||||
|
||||
fps: this.fps,
|
||||
bitrate: this.bitrate,
|
||||
ratio: this.ratio,
|
||||
hasAudio: this.hasAudio,
|
||||
|
||||
segmentListSize: VIDEO_LIVE.SEGMENTS_LIST_SIZE,
|
||||
segmentDuration: getLiveSegmentTime(this.videoLive.latencyMode),
|
||||
|
||||
outDirectory: this.outDirectory
|
||||
}
|
||||
|
||||
return CONFIG.LIVE.TRANSCODING.ENABLED && CONFIG.LIVE.TRANSCODING.REMOTE_RUNNERS.ENABLED
|
||||
? new RemoteTranscodingWrapper(options)
|
||||
: new FFmpegTranscodingWrapper(options)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
@ -0,0 +1,101 @@
|
||||
import EventEmitter from 'events'
|
||||
import { LoggerTagsFn } from '@server/helpers/logger'
|
||||
import { MStreamingPlaylistVideo, MVideoLiveVideo } from '@server/types/models'
|
||||
import { LiveVideoError } from '@shared/models'
|
||||
|
||||
interface TranscodingWrapperEvents {
|
||||
'end': () => void
|
||||
|
||||
'error': (options: { err: Error }) => void
|
||||
}
|
||||
|
||||
declare interface AbstractTranscodingWrapper {
|
||||
on<U extends keyof TranscodingWrapperEvents>(
|
||||
event: U, listener: TranscodingWrapperEvents[U]
|
||||
): this
|
||||
|
||||
emit<U extends keyof TranscodingWrapperEvents>(
|
||||
event: U, ...args: Parameters<TranscodingWrapperEvents[U]>
|
||||
): boolean
|
||||
}
|
||||
|
||||
interface AbstractTranscodingWrapperOptions {
|
||||
streamingPlaylist: MStreamingPlaylistVideo
|
||||
videoLive: MVideoLiveVideo
|
||||
|
||||
lTags: LoggerTagsFn
|
||||
|
||||
inputUrl: string
|
||||
fps: number
|
||||
toTranscode: {
|
||||
resolution: number
|
||||
fps: number
|
||||
}[]
|
||||
|
||||
bitrate: number
|
||||
ratio: number
|
||||
hasAudio: boolean
|
||||
|
||||
segmentListSize: number
|
||||
segmentDuration: number
|
||||
|
||||
outDirectory: string
|
||||
}
|
||||
|
||||
abstract class AbstractTranscodingWrapper extends EventEmitter {
|
||||
protected readonly videoLive: MVideoLiveVideo
|
||||
|
||||
protected readonly toTranscode: {
|
||||
resolution: number
|
||||
fps: number
|
||||
}[]
|
||||
|
||||
protected readonly inputUrl: string
|
||||
protected readonly fps: number
|
||||
protected readonly bitrate: number
|
||||
protected readonly ratio: number
|
||||
protected readonly hasAudio: boolean
|
||||
|
||||
protected readonly segmentListSize: number
|
||||
protected readonly segmentDuration: number
|
||||
|
||||
protected readonly videoUUID: string
|
||||
|
||||
protected readonly outDirectory: string
|
||||
|
||||
protected readonly lTags: LoggerTagsFn
|
||||
|
||||
protected readonly streamingPlaylist: MStreamingPlaylistVideo
|
||||
|
||||
constructor (options: AbstractTranscodingWrapperOptions) {
|
||||
super()
|
||||
|
||||
this.lTags = options.lTags
|
||||
|
||||
this.videoLive = options.videoLive
|
||||
this.videoUUID = options.videoLive.Video.uuid
|
||||
this.streamingPlaylist = options.streamingPlaylist
|
||||
|
||||
this.inputUrl = options.inputUrl
|
||||
this.fps = options.fps
|
||||
this.toTranscode = options.toTranscode
|
||||
|
||||
this.bitrate = options.bitrate
|
||||
this.ratio = options.ratio
|
||||
this.hasAudio = options.hasAudio
|
||||
|
||||
this.segmentListSize = options.segmentListSize
|
||||
this.segmentDuration = options.segmentDuration
|
||||
|
||||
this.outDirectory = options.outDirectory
|
||||
}
|
||||
|
||||
abstract run (): Promise<void>
|
||||
|
||||
abstract abort (error?: LiveVideoError): void
|
||||
}
|
||||
|
||||
export {
|
||||
AbstractTranscodingWrapper,
|
||||
AbstractTranscodingWrapperOptions
|
||||
}
|
@ -0,0 +1,95 @@
|
||||
import { FfmpegCommand } from 'fluent-ffmpeg'
|
||||
import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg'
|
||||
import { logger } from '@server/helpers/logger'
|
||||
import { CONFIG } from '@server/initializers/config'
|
||||
import { VIDEO_LIVE } from '@server/initializers/constants'
|
||||
import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles'
|
||||
import { FFmpegLive } from '@shared/ffmpeg'
|
||||
import { getLiveSegmentTime } from '../../live-utils'
|
||||
import { AbstractTranscodingWrapper } from './abstract-transcoding-wrapper'
|
||||
|
||||
export class FFmpegTranscodingWrapper extends AbstractTranscodingWrapper {
|
||||
private ffmpegCommand: FfmpegCommand
|
||||
private ended = false
|
||||
|
||||
async run () {
|
||||
this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED
|
||||
? await this.buildFFmpegLive().getLiveTranscodingCommand({
|
||||
inputUrl: this.inputUrl,
|
||||
|
||||
outPath: this.outDirectory,
|
||||
masterPlaylistName: this.streamingPlaylist.playlistFilename,
|
||||
|
||||
segmentListSize: this.segmentListSize,
|
||||
segmentDuration: this.segmentDuration,
|
||||
|
||||
toTranscode: this.toTranscode,
|
||||
|
||||
bitrate: this.bitrate,
|
||||
ratio: this.ratio,
|
||||
|
||||
hasAudio: this.hasAudio
|
||||
})
|
||||
: this.buildFFmpegLive().getLiveMuxingCommand({
|
||||
inputUrl: this.inputUrl,
|
||||
outPath: this.outDirectory,
|
||||
|
||||
masterPlaylistName: this.streamingPlaylist.playlistFilename,
|
||||
|
||||
segmentListSize: VIDEO_LIVE.SEGMENTS_LIST_SIZE,
|
||||
segmentDuration: getLiveSegmentTime(this.videoLive.latencyMode)
|
||||
})
|
||||
|
||||
logger.info('Running local live muxing/transcoding for %s.', this.videoUUID, this.lTags())
|
||||
|
||||
this.ffmpegCommand.run()
|
||||
|
||||
let ffmpegShellCommand: string
|
||||
this.ffmpegCommand.on('start', cmdline => {
|
||||
ffmpegShellCommand = cmdline
|
||||
|
||||
logger.debug('Running ffmpeg command for live', { ffmpegShellCommand, ...this.lTags() })
|
||||
})
|
||||
|
||||
this.ffmpegCommand.on('error', (err, stdout, stderr) => {
|
||||
this.onFFmpegError({ err, stdout, stderr, ffmpegShellCommand })
|
||||
})
|
||||
|
||||
this.ffmpegCommand.on('end', () => {
|
||||
this.onFFmpegEnded()
|
||||
})
|
||||
|
||||
this.ffmpegCommand.run()
|
||||
}
|
||||
|
||||
abort () {
|
||||
// Nothing to do, ffmpeg will automatically exit
|
||||
}
|
||||
|
||||
private onFFmpegError (options: {
|
||||
err: any
|
||||
stdout: string
|
||||
stderr: string
|
||||
ffmpegShellCommand: string
|
||||
}) {
|
||||
const { err, stdout, stderr, ffmpegShellCommand } = options
|
||||
|
||||
// Don't care that we killed the ffmpeg process
|
||||
if (err?.message?.includes('Exiting normally')) return
|
||||
|
||||
logger.error('FFmpeg transcoding error.', { err, stdout, stderr, ffmpegShellCommand, ...this.lTags() })
|
||||
|
||||
this.emit('error', { err })
|
||||
}
|
||||
|
||||
private onFFmpegEnded () {
|
||||
if (this.ended) return
|
||||
|
||||
this.ended = true
|
||||
this.emit('end')
|
||||
}
|
||||
|
||||
private buildFFmpegLive () {
|
||||
return new FFmpegLive(getFFmpegCommandWrapperOptions('live', VideoTranscodingProfilesManager.Instance.getAvailableEncoders()))
|
||||
}
|
||||
}
|
3
server/lib/live/shared/transcoding-wrapper/index.ts
Normal file
3
server/lib/live/shared/transcoding-wrapper/index.ts
Normal file
@ -0,0 +1,3 @@
|
||||
export * from './abstract-transcoding-wrapper'
|
||||
export * from './ffmpeg-transcoding-wrapper'
|
||||
export * from './remote-transcoding-wrapper'
|
@ -0,0 +1,20 @@
|
||||
import { LiveRTMPHLSTranscodingJobHandler } from '@server/lib/runners'
|
||||
import { AbstractTranscodingWrapper } from './abstract-transcoding-wrapper'
|
||||
|
||||
export class RemoteTranscodingWrapper extends AbstractTranscodingWrapper {
|
||||
async run () {
|
||||
await new LiveRTMPHLSTranscodingJobHandler().create({
|
||||
rtmpUrl: this.inputUrl,
|
||||
toTranscode: this.toTranscode,
|
||||
video: this.videoLive.Video,
|
||||
outputDirectory: this.outDirectory,
|
||||
playlist: this.streamingPlaylist,
|
||||
segmentListSize: this.segmentListSize,
|
||||
segmentDuration: this.segmentDuration
|
||||
})
|
||||
}
|
||||
|
||||
abort () {
|
||||
this.emit('end')
|
||||
}
|
||||
}
|
@ -1,3 +1,4 @@
|
||||
export * from './keys'
|
||||
export * from './proxy'
|
||||
export * from './urls'
|
||||
export * from './videos'
|
||||
|
97
server/lib/object-storage/proxy.ts
Normal file
97
server/lib/object-storage/proxy.ts
Normal file
@ -0,0 +1,97 @@
|
||||
import express from 'express'
|
||||
import { PassThrough, pipeline } from 'stream'
|
||||
import { GetObjectCommandOutput } from '@aws-sdk/client-s3'
|
||||
import { buildReinjectVideoFileTokenQuery } from '@server/controllers/shared/m3u8-playlist'
|
||||
import { logger } from '@server/helpers/logger'
|
||||
import { StreamReplacer } from '@server/helpers/stream-replacer'
|
||||
import { MStreamingPlaylist, MVideo } from '@server/types/models'
|
||||
import { HttpStatusCode } from '@shared/models'
|
||||
import { injectQueryToPlaylistUrls } from '../hls'
|
||||
import { getHLSFileReadStream, getWebTorrentFileReadStream } from './videos'
|
||||
|
||||
export async function proxifyWebTorrentFile (options: {
|
||||
req: express.Request
|
||||
res: express.Response
|
||||
filename: string
|
||||
}) {
|
||||
const { req, res, filename } = options
|
||||
|
||||
logger.debug('Proxifying WebTorrent file %s from object storage.', filename)
|
||||
|
||||
try {
|
||||
const { response: s3Response, stream } = await getWebTorrentFileReadStream({
|
||||
filename,
|
||||
rangeHeader: req.header('range')
|
||||
})
|
||||
|
||||
setS3Headers(res, s3Response)
|
||||
|
||||
return stream.pipe(res)
|
||||
} catch (err) {
|
||||
return handleObjectStorageFailure(res, err)
|
||||
}
|
||||
}
|
||||
|
||||
export async function proxifyHLS (options: {
|
||||
req: express.Request
|
||||
res: express.Response
|
||||
playlist: MStreamingPlaylist
|
||||
video: MVideo
|
||||
filename: string
|
||||
reinjectVideoFileToken: boolean
|
||||
}) {
|
||||
const { req, res, playlist, video, filename, reinjectVideoFileToken } = options
|
||||
|
||||
logger.debug('Proxifying HLS file %s from object storage.', filename)
|
||||
|
||||
try {
|
||||
const { response: s3Response, stream } = await getHLSFileReadStream({
|
||||
playlist: playlist.withVideo(video),
|
||||
filename,
|
||||
rangeHeader: req.header('range')
|
||||
})
|
||||
|
||||
setS3Headers(res, s3Response)
|
||||
|
||||
const streamReplacer = reinjectVideoFileToken
|
||||
? new StreamReplacer(line => injectQueryToPlaylistUrls(line, buildReinjectVideoFileTokenQuery(req, filename.endsWith('master.m3u8'))))
|
||||
: new PassThrough()
|
||||
|
||||
return pipeline(
|
||||
stream,
|
||||
streamReplacer,
|
||||
res,
|
||||
err => {
|
||||
if (!err) return
|
||||
|
||||
handleObjectStorageFailure(res, err)
|
||||
}
|
||||
)
|
||||
} catch (err) {
|
||||
return handleObjectStorageFailure(res, err)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Private
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function handleObjectStorageFailure (res: express.Response, err: Error) {
|
||||
if (err.name === 'NoSuchKey') {
|
||||
logger.debug('Could not find key in object storage to proxify private HLS video file.', { err })
|
||||
return res.sendStatus(HttpStatusCode.NOT_FOUND_404)
|
||||
}
|
||||
|
||||
return res.fail({
|
||||
status: HttpStatusCode.INTERNAL_SERVER_ERROR_500,
|
||||
message: err.message,
|
||||
type: err.name
|
||||
})
|
||||
}
|
||||
|
||||
function setS3Headers (res: express.Response, s3Response: GetObjectCommandOutput) {
|
||||
if (s3Response.$metadata.httpStatusCode === HttpStatusCode.PARTIAL_CONTENT_206) {
|
||||
res.setHeader('Content-Range', s3Response.ContentRange)
|
||||
res.status(HttpStatusCode.PARTIAL_CONTENT_206)
|
||||
}
|
||||
}
|
@ -2,10 +2,12 @@ import { Server as HTTPServer } from 'http'
|
||||
import { Namespace, Server as SocketServer, Socket } from 'socket.io'
|
||||
import { isIdValid } from '@server/helpers/custom-validators/misc'
|
||||
import { MVideo, MVideoImmutable } from '@server/types/models'
|
||||
import { MRunner } from '@server/types/models/runners'
|
||||
import { UserNotificationModelForApi } from '@server/types/models/user'
|
||||
import { LiveVideoEventPayload, LiveVideoEventType } from '@shared/models'
|
||||
import { logger } from '../helpers/logger'
|
||||
import { authenticateSocket } from '../middlewares'
|
||||
import { authenticateRunnerSocket, authenticateSocket } from '../middlewares'
|
||||
import { Debounce } from '@server/helpers/debounce'
|
||||
|
||||
class PeerTubeSocket {
|
||||
|
||||
@ -13,6 +15,7 @@ class PeerTubeSocket {
|
||||
|
||||
private userNotificationSockets: { [ userId: number ]: Socket[] } = {}
|
||||
private liveVideosNamespace: Namespace
|
||||
private readonly runnerSockets = new Set<Socket>()
|
||||
|
||||
private constructor () {}
|
||||
|
||||
@ -24,7 +27,7 @@ class PeerTubeSocket {
|
||||
.on('connection', socket => {
|
||||
const userId = socket.handshake.auth.user.id
|
||||
|
||||
logger.debug('User %d connected on the notification system.', userId)
|
||||
logger.debug('User %d connected to the notification system.', userId)
|
||||
|
||||
if (!this.userNotificationSockets[userId]) this.userNotificationSockets[userId] = []
|
||||
|
||||
@ -53,6 +56,22 @@ class PeerTubeSocket {
|
||||
socket.leave(videoId)
|
||||
})
|
||||
})
|
||||
|
||||
io.of('/runners')
|
||||
.use(authenticateRunnerSocket)
|
||||
.on('connection', socket => {
|
||||
const runner: MRunner = socket.handshake.auth.runner
|
||||
|
||||
logger.debug(`New runner "${runner.name}" connected to the notification system.`)
|
||||
|
||||
this.runnerSockets.add(socket)
|
||||
|
||||
socket.on('disconnect', () => {
|
||||
logger.debug(`Runner "${runner.name}" disconnected from the notification system.`)
|
||||
|
||||
this.runnerSockets.delete(socket)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
sendNotification (userId: number, notification: UserNotificationModelForApi) {
|
||||
@ -89,6 +108,15 @@ class PeerTubeSocket {
|
||||
.emit(type, data)
|
||||
}
|
||||
|
||||
@Debounce({ timeoutMS: 1000 })
|
||||
sendAvailableJobsPingToRunners () {
|
||||
logger.debug(`Sending available-jobs notification to ${this.runnerSockets.size} runner sockets`)
|
||||
|
||||
for (const runners of this.runnerSockets) {
|
||||
runners.emit('available-jobs')
|
||||
}
|
||||
}
|
||||
|
||||
static get Instance () {
|
||||
return this.instance || (this.instance = new this())
|
||||
}
|
||||
|
@ -1,7 +1,6 @@
|
||||
import express from 'express'
|
||||
import { Server } from 'http'
|
||||
import { join } from 'path'
|
||||
import { ffprobePromise } from '@server/helpers/ffmpeg/ffprobe-utils'
|
||||
import { buildLogger } from '@server/helpers/logger'
|
||||
import { CONFIG } from '@server/initializers/config'
|
||||
import { WEBSERVER } from '@server/initializers/constants'
|
||||
@ -16,6 +15,7 @@ import { VideoModel } from '@server/models/video/video'
|
||||
import { VideoBlacklistModel } from '@server/models/video/video-blacklist'
|
||||
import { MPlugin, MVideo, UserNotificationModelForApi } from '@server/types/models'
|
||||
import { PeerTubeHelpers } from '@server/types/plugins'
|
||||
import { ffprobePromise } from '@shared/ffmpeg'
|
||||
import { VideoBlacklistCreate, VideoStorage } from '@shared/models'
|
||||
import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist'
|
||||
import { PeerTubeSocket } from '../peertube-socket'
|
||||
|
3
server/lib/runners/index.ts
Normal file
3
server/lib/runners/index.ts
Normal file
@ -0,0 +1,3 @@
|
||||
export * from './job-handlers'
|
||||
export * from './runner'
|
||||
export * from './runner-urls'
|
271
server/lib/runners/job-handlers/abstract-job-handler.ts
Normal file
271
server/lib/runners/job-handlers/abstract-job-handler.ts
Normal file
@ -0,0 +1,271 @@
|
||||
import { retryTransactionWrapper } from '@server/helpers/database-utils'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger'
|
||||
import { RUNNER_JOBS } from '@server/initializers/constants'
|
||||
import { sequelizeTypescript } from '@server/initializers/database'
|
||||
import { PeerTubeSocket } from '@server/lib/peertube-socket'
|
||||
import { RunnerJobModel } from '@server/models/runner/runner-job'
|
||||
import { setAsUpdated } from '@server/models/shared'
|
||||
import { MRunnerJob } from '@server/types/models/runners'
|
||||
import { pick } from '@shared/core-utils'
|
||||
import {
|
||||
RunnerJobLiveRTMPHLSTranscodingPayload,
|
||||
RunnerJobLiveRTMPHLSTranscodingPrivatePayload,
|
||||
RunnerJobState,
|
||||
RunnerJobSuccessPayload,
|
||||
RunnerJobType,
|
||||
RunnerJobUpdatePayload,
|
||||
RunnerJobVODAudioMergeTranscodingPayload,
|
||||
RunnerJobVODAudioMergeTranscodingPrivatePayload,
|
||||
RunnerJobVODHLSTranscodingPayload,
|
||||
RunnerJobVODHLSTranscodingPrivatePayload,
|
||||
RunnerJobVODWebVideoTranscodingPayload,
|
||||
RunnerJobVODWebVideoTranscodingPrivatePayload
|
||||
} from '@shared/models'
|
||||
|
||||
type CreateRunnerJobArg =
|
||||
{
|
||||
type: Extract<RunnerJobType, 'vod-web-video-transcoding'>
|
||||
payload: RunnerJobVODWebVideoTranscodingPayload
|
||||
privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload
|
||||
} |
|
||||
{
|
||||
type: Extract<RunnerJobType, 'vod-hls-transcoding'>
|
||||
payload: RunnerJobVODHLSTranscodingPayload
|
||||
privatePayload: RunnerJobVODHLSTranscodingPrivatePayload
|
||||
} |
|
||||
{
|
||||
type: Extract<RunnerJobType, 'vod-audio-merge-transcoding'>
|
||||
payload: RunnerJobVODAudioMergeTranscodingPayload
|
||||
privatePayload: RunnerJobVODAudioMergeTranscodingPrivatePayload
|
||||
} |
|
||||
{
|
||||
type: Extract<RunnerJobType, 'live-rtmp-hls-transcoding'>
|
||||
payload: RunnerJobLiveRTMPHLSTranscodingPayload
|
||||
privatePayload: RunnerJobLiveRTMPHLSTranscodingPrivatePayload
|
||||
}
|
||||
|
||||
export abstract class AbstractJobHandler <C, U extends RunnerJobUpdatePayload, S extends RunnerJobSuccessPayload> {
|
||||
|
||||
protected readonly lTags = loggerTagsFactory('runner')
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
abstract create (options: C): Promise<MRunnerJob>
|
||||
|
||||
protected async createRunnerJob (options: CreateRunnerJobArg & {
|
||||
jobUUID: string
|
||||
priority: number
|
||||
dependsOnRunnerJob?: MRunnerJob
|
||||
}): Promise<MRunnerJob> {
|
||||
const { priority, dependsOnRunnerJob } = options
|
||||
|
||||
const runnerJob = new RunnerJobModel({
|
||||
...pick(options, [ 'type', 'payload', 'privatePayload' ]),
|
||||
|
||||
uuid: options.jobUUID,
|
||||
|
||||
state: dependsOnRunnerJob
|
||||
? RunnerJobState.WAITING_FOR_PARENT_JOB
|
||||
: RunnerJobState.PENDING,
|
||||
|
||||
dependsOnRunnerJobId: dependsOnRunnerJob?.id,
|
||||
|
||||
priority
|
||||
})
|
||||
|
||||
const job = await sequelizeTypescript.transaction(async transaction => {
|
||||
return runnerJob.save({ transaction })
|
||||
})
|
||||
|
||||
if (runnerJob.state === RunnerJobState.PENDING) {
|
||||
PeerTubeSocket.Instance.sendAvailableJobsPingToRunners()
|
||||
}
|
||||
|
||||
return job
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
protected abstract specificUpdate (options: {
|
||||
runnerJob: MRunnerJob
|
||||
updatePayload?: U
|
||||
}): Promise<void> | void
|
||||
|
||||
async update (options: {
|
||||
runnerJob: MRunnerJob
|
||||
progress?: number
|
||||
updatePayload?: U
|
||||
}) {
|
||||
const { runnerJob, progress } = options
|
||||
|
||||
await this.specificUpdate(options)
|
||||
|
||||
if (progress) runnerJob.progress = progress
|
||||
|
||||
await retryTransactionWrapper(() => {
|
||||
return sequelizeTypescript.transaction(async transaction => {
|
||||
if (runnerJob.changed()) {
|
||||
return runnerJob.save({ transaction })
|
||||
}
|
||||
|
||||
// Don't update the job too often
|
||||
if (new Date().getTime() - runnerJob.updatedAt.getTime() > 2000) {
|
||||
await setAsUpdated({ sequelize: sequelizeTypescript, table: 'runnerJob', id: runnerJob.id, transaction })
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async complete (options: {
|
||||
runnerJob: MRunnerJob
|
||||
resultPayload: S
|
||||
}) {
|
||||
const { runnerJob } = options
|
||||
|
||||
try {
|
||||
await this.specificComplete(options)
|
||||
|
||||
runnerJob.state = RunnerJobState.COMPLETED
|
||||
} catch (err) {
|
||||
logger.error('Cannot complete runner job', { err, ...this.lTags(runnerJob.id, runnerJob.type) })
|
||||
|
||||
runnerJob.state = RunnerJobState.ERRORED
|
||||
runnerJob.error = err.message
|
||||
}
|
||||
|
||||
runnerJob.progress = null
|
||||
runnerJob.finishedAt = new Date()
|
||||
|
||||
await retryTransactionWrapper(() => {
|
||||
return sequelizeTypescript.transaction(async transaction => {
|
||||
await runnerJob.save({ transaction })
|
||||
})
|
||||
})
|
||||
|
||||
const [ affectedCount ] = await RunnerJobModel.updateDependantJobsOf(runnerJob)
|
||||
|
||||
if (affectedCount !== 0) PeerTubeSocket.Instance.sendAvailableJobsPingToRunners()
|
||||
}
|
||||
|
||||
protected abstract specificComplete (options: {
|
||||
runnerJob: MRunnerJob
|
||||
resultPayload: S
|
||||
}): Promise<void> | void
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async cancel (options: {
|
||||
runnerJob: MRunnerJob
|
||||
fromParent?: boolean
|
||||
}) {
|
||||
const { runnerJob, fromParent } = options
|
||||
|
||||
await this.specificCancel(options)
|
||||
|
||||
const cancelState = fromParent
|
||||
? RunnerJobState.PARENT_CANCELLED
|
||||
: RunnerJobState.CANCELLED
|
||||
|
||||
runnerJob.setToErrorOrCancel(cancelState)
|
||||
|
||||
await retryTransactionWrapper(() => {
|
||||
return sequelizeTypescript.transaction(async transaction => {
|
||||
await runnerJob.save({ transaction })
|
||||
})
|
||||
})
|
||||
|
||||
const children = await RunnerJobModel.listChildrenOf(runnerJob)
|
||||
for (const child of children) {
|
||||
logger.info(`Cancelling child job ${child.uuid} of ${runnerJob.uuid} because of parent cancel`, this.lTags(child.uuid))
|
||||
|
||||
await this.cancel({ runnerJob: child, fromParent: true })
|
||||
}
|
||||
}
|
||||
|
||||
protected abstract specificCancel (options: {
|
||||
runnerJob: MRunnerJob
|
||||
}): Promise<void> | void
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
protected abstract isAbortSupported (): boolean
|
||||
|
||||
async abort (options: {
|
||||
runnerJob: MRunnerJob
|
||||
}) {
|
||||
const { runnerJob } = options
|
||||
|
||||
if (this.isAbortSupported() !== true) {
|
||||
return this.error({ runnerJob, message: 'Job has been aborted but it is not supported by this job type' })
|
||||
}
|
||||
|
||||
await this.specificAbort(options)
|
||||
|
||||
runnerJob.resetToPending()
|
||||
|
||||
await retryTransactionWrapper(() => {
|
||||
return sequelizeTypescript.transaction(async transaction => {
|
||||
await runnerJob.save({ transaction })
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
protected setAbortState (runnerJob: MRunnerJob) {
|
||||
runnerJob.resetToPending()
|
||||
}
|
||||
|
||||
protected abstract specificAbort (options: {
|
||||
runnerJob: MRunnerJob
|
||||
}): Promise<void> | void
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async error (options: {
|
||||
runnerJob: MRunnerJob
|
||||
message: string
|
||||
fromParent?: boolean
|
||||
}) {
|
||||
const { runnerJob, message, fromParent } = options
|
||||
|
||||
const errorState = fromParent
|
||||
? RunnerJobState.PARENT_ERRORED
|
||||
: RunnerJobState.ERRORED
|
||||
|
||||
const nextState = errorState === RunnerJobState.ERRORED && this.isAbortSupported() && runnerJob.failures < RUNNER_JOBS.MAX_FAILURES
|
||||
? RunnerJobState.PENDING
|
||||
: errorState
|
||||
|
||||
await this.specificError({ ...options, nextState })
|
||||
|
||||
if (nextState === errorState) {
|
||||
runnerJob.setToErrorOrCancel(nextState)
|
||||
runnerJob.error = message
|
||||
} else {
|
||||
runnerJob.resetToPending()
|
||||
}
|
||||
|
||||
await retryTransactionWrapper(() => {
|
||||
return sequelizeTypescript.transaction(async transaction => {
|
||||
await runnerJob.save({ transaction })
|
||||
})
|
||||
})
|
||||
|
||||
if (runnerJob.state === errorState) {
|
||||
const children = await RunnerJobModel.listChildrenOf(runnerJob)
|
||||
|
||||
for (const child of children) {
|
||||
logger.info(`Erroring child job ${child.uuid} of ${runnerJob.uuid} because of parent error`, this.lTags(child.uuid))
|
||||
|
||||
await this.error({ runnerJob: child, message: 'Parent error', fromParent: true })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected abstract specificError (options: {
|
||||
runnerJob: MRunnerJob
|
||||
message: string
|
||||
nextState: RunnerJobState
|
||||
}): Promise<void> | void
|
||||
}
|
@ -0,0 +1,71 @@
|
||||
|
||||
import { retryTransactionWrapper } from '@server/helpers/database-utils'
|
||||
import { logger } from '@server/helpers/logger'
|
||||
import { moveToFailedTranscodingState, moveToNextState } from '@server/lib/video-state'
|
||||
import { VideoJobInfoModel } from '@server/models/video/video-job-info'
|
||||
import { MRunnerJob } from '@server/types/models/runners'
|
||||
import {
|
||||
LiveRTMPHLSTranscodingUpdatePayload,
|
||||
RunnerJobSuccessPayload,
|
||||
RunnerJobUpdatePayload,
|
||||
RunnerJobVODPrivatePayload
|
||||
} from '@shared/models'
|
||||
import { AbstractJobHandler } from './abstract-job-handler'
|
||||
import { loadTranscodingRunnerVideo } from './shared'
|
||||
|
||||
// eslint-disable-next-line max-len
|
||||
export abstract class AbstractVODTranscodingJobHandler <C, U extends RunnerJobUpdatePayload, S extends RunnerJobSuccessPayload> extends AbstractJobHandler<C, U, S> {
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
protected isAbortSupported () {
|
||||
return true
|
||||
}
|
||||
|
||||
protected specificUpdate (_options: {
|
||||
runnerJob: MRunnerJob
|
||||
updatePayload?: LiveRTMPHLSTranscodingUpdatePayload
|
||||
}) {
|
||||
// empty
|
||||
}
|
||||
|
||||
protected specificAbort (_options: {
|
||||
runnerJob: MRunnerJob
|
||||
}) {
|
||||
// empty
|
||||
}
|
||||
|
||||
protected async specificError (options: {
|
||||
runnerJob: MRunnerJob
|
||||
}) {
|
||||
const video = await loadTranscodingRunnerVideo(options.runnerJob, this.lTags)
|
||||
if (!video) return
|
||||
|
||||
await moveToFailedTranscodingState(video)
|
||||
|
||||
await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
|
||||
}
|
||||
|
||||
protected async specificCancel (options: {
|
||||
runnerJob: MRunnerJob
|
||||
}) {
|
||||
const { runnerJob } = options
|
||||
|
||||
const video = await loadTranscodingRunnerVideo(options.runnerJob, this.lTags)
|
||||
if (!video) return
|
||||
|
||||
const pending = await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
|
||||
|
||||
logger.debug(`Pending transcode decreased to ${pending} after cancel`, this.lTags(video.uuid))
|
||||
|
||||
if (pending === 0) {
|
||||
logger.info(
|
||||
`All transcoding jobs of ${video.uuid} have been processed or canceled, moving it to its next state`,
|
||||
this.lTags(video.uuid)
|
||||
)
|
||||
|
||||
const privatePayload = runnerJob.privatePayload as RunnerJobVODPrivatePayload
|
||||
await retryTransactionWrapper(moveToNextState, { video, isNewVideo: privatePayload.isNewVideo })
|
||||
}
|
||||
}
|
||||
}
|
6
server/lib/runners/job-handlers/index.ts
Normal file
6
server/lib/runners/job-handlers/index.ts
Normal file
@ -0,0 +1,6 @@
|
||||
export * from './abstract-job-handler'
|
||||
export * from './live-rtmp-hls-transcoding-job-handler'
|
||||
export * from './vod-audio-merge-transcoding-job-handler'
|
||||
export * from './vod-hls-transcoding-job-handler'
|
||||
export * from './vod-web-video-transcoding-job-handler'
|
||||
export * from './runner-job-handlers'
|
@ -0,0 +1,170 @@
|
||||
import { move, remove } from 'fs-extra'
|
||||
import { join } from 'path'
|
||||
import { logger } from '@server/helpers/logger'
|
||||
import { JOB_PRIORITY } from '@server/initializers/constants'
|
||||
import { LiveManager } from '@server/lib/live'
|
||||
import { MStreamingPlaylist, MVideo } from '@server/types/models'
|
||||
import { MRunnerJob } from '@server/types/models/runners'
|
||||
import { buildUUID } from '@shared/extra-utils'
|
||||
import {
|
||||
LiveRTMPHLSTranscodingSuccess,
|
||||
LiveRTMPHLSTranscodingUpdatePayload,
|
||||
LiveVideoError,
|
||||
RunnerJobLiveRTMPHLSTranscodingPayload,
|
||||
RunnerJobLiveRTMPHLSTranscodingPrivatePayload,
|
||||
RunnerJobState
|
||||
} from '@shared/models'
|
||||
import { AbstractJobHandler } from './abstract-job-handler'
|
||||
|
||||
type CreateOptions = {
|
||||
video: MVideo
|
||||
playlist: MStreamingPlaylist
|
||||
|
||||
rtmpUrl: string
|
||||
|
||||
toTranscode: {
|
||||
resolution: number
|
||||
fps: number
|
||||
}[]
|
||||
|
||||
segmentListSize: number
|
||||
segmentDuration: number
|
||||
|
||||
outputDirectory: string
|
||||
}
|
||||
|
||||
// eslint-disable-next-line max-len
|
||||
export class LiveRTMPHLSTranscodingJobHandler extends AbstractJobHandler<CreateOptions, LiveRTMPHLSTranscodingUpdatePayload, LiveRTMPHLSTranscodingSuccess> {
|
||||
|
||||
async create (options: CreateOptions) {
|
||||
const { video, rtmpUrl, toTranscode, playlist, segmentDuration, segmentListSize, outputDirectory } = options
|
||||
|
||||
const jobUUID = buildUUID()
|
||||
const payload: RunnerJobLiveRTMPHLSTranscodingPayload = {
|
||||
input: {
|
||||
rtmpUrl
|
||||
},
|
||||
output: {
|
||||
toTranscode,
|
||||
segmentListSize,
|
||||
segmentDuration
|
||||
}
|
||||
}
|
||||
|
||||
const privatePayload: RunnerJobLiveRTMPHLSTranscodingPrivatePayload = {
|
||||
videoUUID: video.uuid,
|
||||
masterPlaylistName: playlist.playlistFilename,
|
||||
outputDirectory
|
||||
}
|
||||
|
||||
const job = await this.createRunnerJob({
|
||||
type: 'live-rtmp-hls-transcoding',
|
||||
jobUUID,
|
||||
payload,
|
||||
privatePayload,
|
||||
priority: JOB_PRIORITY.TRANSCODING
|
||||
})
|
||||
|
||||
return job
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async specificUpdate (options: {
|
||||
runnerJob: MRunnerJob
|
||||
updatePayload: LiveRTMPHLSTranscodingUpdatePayload
|
||||
}) {
|
||||
const { runnerJob, updatePayload } = options
|
||||
|
||||
const privatePayload = runnerJob.privatePayload as RunnerJobLiveRTMPHLSTranscodingPrivatePayload
|
||||
const outputDirectory = privatePayload.outputDirectory
|
||||
const videoUUID = privatePayload.videoUUID
|
||||
|
||||
if (updatePayload.type === 'add-chunk') {
|
||||
await move(
|
||||
updatePayload.videoChunkFile as string,
|
||||
join(outputDirectory, updatePayload.videoChunkFilename),
|
||||
{ overwrite: true }
|
||||
)
|
||||
} else if (updatePayload.type === 'remove-chunk') {
|
||||
await remove(join(outputDirectory, updatePayload.videoChunkFilename))
|
||||
}
|
||||
|
||||
if (updatePayload.resolutionPlaylistFile && updatePayload.resolutionPlaylistFilename) {
|
||||
await move(
|
||||
updatePayload.resolutionPlaylistFile as string,
|
||||
join(outputDirectory, updatePayload.resolutionPlaylistFilename),
|
||||
{ overwrite: true }
|
||||
)
|
||||
}
|
||||
|
||||
if (updatePayload.masterPlaylistFile) {
|
||||
await move(updatePayload.masterPlaylistFile as string, join(outputDirectory, privatePayload.masterPlaylistName), { overwrite: true })
|
||||
}
|
||||
|
||||
logger.info(
|
||||
'Runner live RTMP to HLS job %s for %s updated.',
|
||||
runnerJob.uuid, videoUUID, { updatePayload, ...this.lTags(videoUUID, runnerJob.uuid) }
|
||||
)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
protected specificComplete (options: {
|
||||
runnerJob: MRunnerJob
|
||||
}) {
|
||||
return this.stopLive({
|
||||
runnerJob: options.runnerJob,
|
||||
type: 'ended'
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
protected isAbortSupported () {
|
||||
return false
|
||||
}
|
||||
|
||||
protected specificAbort () {
|
||||
throw new Error('Not implemented')
|
||||
}
|
||||
|
||||
protected specificError (options: {
|
||||
runnerJob: MRunnerJob
|
||||
nextState: RunnerJobState
|
||||
}) {
|
||||
return this.stopLive({
|
||||
runnerJob: options.runnerJob,
|
||||
type: 'errored'
|
||||
})
|
||||
}
|
||||
|
||||
protected specificCancel (options: {
|
||||
runnerJob: MRunnerJob
|
||||
}) {
|
||||
return this.stopLive({
|
||||
runnerJob: options.runnerJob,
|
||||
type: 'cancelled'
|
||||
})
|
||||
}
|
||||
|
||||
private stopLive (options: {
|
||||
runnerJob: MRunnerJob
|
||||
type: 'ended' | 'errored' | 'cancelled'
|
||||
}) {
|
||||
const { runnerJob, type } = options
|
||||
|
||||
const privatePayload = runnerJob.privatePayload as RunnerJobLiveRTMPHLSTranscodingPrivatePayload
|
||||
const videoUUID = privatePayload.videoUUID
|
||||
|
||||
const errorType = {
|
||||
ended: null,
|
||||
errored: LiveVideoError.RUNNER_JOB_ERROR,
|
||||
cancelled: LiveVideoError.RUNNER_JOB_CANCEL
|
||||
}
|
||||
|
||||
LiveManager.Instance.stopSessionOf(privatePayload.videoUUID, errorType[type])
|
||||
|
||||
logger.info('Runner live RTMP to HLS job %s for video %s %s.', runnerJob.uuid, videoUUID, type, this.lTags(runnerJob.uuid, videoUUID))
|
||||
}
|
||||
}
|
18
server/lib/runners/job-handlers/runner-job-handlers.ts
Normal file
18
server/lib/runners/job-handlers/runner-job-handlers.ts
Normal file
@ -0,0 +1,18 @@
|
||||
import { MRunnerJob } from '@server/types/models/runners'
|
||||
import { RunnerJobSuccessPayload, RunnerJobType, RunnerJobUpdatePayload } from '@shared/models'
|
||||
import { AbstractJobHandler } from './abstract-job-handler'
|
||||
import { LiveRTMPHLSTranscodingJobHandler } from './live-rtmp-hls-transcoding-job-handler'
|
||||
import { VODAudioMergeTranscodingJobHandler } from './vod-audio-merge-transcoding-job-handler'
|
||||
import { VODHLSTranscodingJobHandler } from './vod-hls-transcoding-job-handler'
|
||||
import { VODWebVideoTranscodingJobHandler } from './vod-web-video-transcoding-job-handler'
|
||||
|
||||
const processors: Record<RunnerJobType, new() => AbstractJobHandler<unknown, RunnerJobUpdatePayload, RunnerJobSuccessPayload>> = {
|
||||
'vod-web-video-transcoding': VODWebVideoTranscodingJobHandler,
|
||||
'vod-hls-transcoding': VODHLSTranscodingJobHandler,
|
||||
'vod-audio-merge-transcoding': VODAudioMergeTranscodingJobHandler,
|
||||
'live-rtmp-hls-transcoding': LiveRTMPHLSTranscodingJobHandler
|
||||
}
|
||||
|
||||
export function getRunnerJobHandlerClass (job: MRunnerJob) {
|
||||
return processors[job.type]
|
||||
}
|
1
server/lib/runners/job-handlers/shared/index.ts
Normal file
1
server/lib/runners/job-handlers/shared/index.ts
Normal file
@ -0,0 +1 @@
|
||||
export * from './vod-helpers'
|
44
server/lib/runners/job-handlers/shared/vod-helpers.ts
Normal file
44
server/lib/runners/job-handlers/shared/vod-helpers.ts
Normal file
@ -0,0 +1,44 @@
|
||||
import { move } from 'fs-extra'
|
||||
import { dirname, join } from 'path'
|
||||
import { logger, LoggerTagsFn } from '@server/helpers/logger'
|
||||
import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding'
|
||||
import { onWebTorrentVideoFileTranscoding } from '@server/lib/transcoding/web-transcoding'
|
||||
import { buildNewFile } from '@server/lib/video-file'
|
||||
import { VideoModel } from '@server/models/video/video'
|
||||
import { MVideoFullLight } from '@server/types/models'
|
||||
import { MRunnerJob } from '@server/types/models/runners'
|
||||
import { RunnerJobVODAudioMergeTranscodingPrivatePayload, RunnerJobVODWebVideoTranscodingPrivatePayload } from '@shared/models'
|
||||
|
||||
export async function onVODWebVideoOrAudioMergeTranscodingJob (options: {
|
||||
video: MVideoFullLight
|
||||
videoFilePath: string
|
||||
privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload | RunnerJobVODAudioMergeTranscodingPrivatePayload
|
||||
}) {
|
||||
const { video, videoFilePath, privatePayload } = options
|
||||
|
||||
const videoFile = await buildNewFile({ path: videoFilePath, mode: 'web-video' })
|
||||
videoFile.videoId = video.id
|
||||
|
||||
const newVideoFilePath = join(dirname(videoFilePath), videoFile.filename)
|
||||
await move(videoFilePath, newVideoFilePath)
|
||||
|
||||
await onWebTorrentVideoFileTranscoding({
|
||||
video,
|
||||
videoFile,
|
||||
videoOutputPath: newVideoFilePath
|
||||
})
|
||||
|
||||
await onTranscodingEnded({ isNewVideo: privatePayload.isNewVideo, moveVideoToNextState: true, video })
|
||||
}
|
||||
|
||||
export async function loadTranscodingRunnerVideo (runnerJob: MRunnerJob, lTags: LoggerTagsFn) {
|
||||
const videoUUID = runnerJob.privatePayload.videoUUID
|
||||
|
||||
const video = await VideoModel.loadFull(videoUUID)
|
||||
if (!video) {
|
||||
logger.info('Video %s does not exist anymore after transcoding runner job.', videoUUID, lTags(videoUUID))
|
||||
return undefined
|
||||
}
|
||||
|
||||
return video
|
||||
}
|
@ -0,0 +1,97 @@
|
||||
import { pick } from 'lodash'
|
||||
import { logger } from '@server/helpers/logger'
|
||||
import { VideoJobInfoModel } from '@server/models/video/video-job-info'
|
||||
import { MVideo } from '@server/types/models'
|
||||
import { MRunnerJob } from '@server/types/models/runners'
|
||||
import { buildUUID } from '@shared/extra-utils'
|
||||
import { getVideoStreamDuration } from '@shared/ffmpeg'
|
||||
import {
|
||||
RunnerJobUpdatePayload,
|
||||
RunnerJobVODAudioMergeTranscodingPayload,
|
||||
RunnerJobVODWebVideoTranscodingPrivatePayload,
|
||||
VODAudioMergeTranscodingSuccess
|
||||
} from '@shared/models'
|
||||
import { generateRunnerTranscodingVideoInputFileUrl, generateRunnerTranscodingVideoPreviewFileUrl } from '../runner-urls'
|
||||
import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler'
|
||||
import { loadTranscodingRunnerVideo, onVODWebVideoOrAudioMergeTranscodingJob } from './shared'
|
||||
|
||||
type CreateOptions = {
|
||||
video: MVideo
|
||||
isNewVideo: boolean
|
||||
resolution: number
|
||||
fps: number
|
||||
priority: number
|
||||
dependsOnRunnerJob?: MRunnerJob
|
||||
}
|
||||
|
||||
// eslint-disable-next-line max-len
|
||||
export class VODAudioMergeTranscodingJobHandler extends AbstractVODTranscodingJobHandler<CreateOptions, RunnerJobUpdatePayload, VODAudioMergeTranscodingSuccess> {
|
||||
|
||||
async create (options: CreateOptions) {
|
||||
const { video, resolution, fps, priority, dependsOnRunnerJob } = options
|
||||
|
||||
const jobUUID = buildUUID()
|
||||
const payload: RunnerJobVODAudioMergeTranscodingPayload = {
|
||||
input: {
|
||||
audioFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid),
|
||||
previewFileUrl: generateRunnerTranscodingVideoPreviewFileUrl(jobUUID, video.uuid)
|
||||
},
|
||||
output: {
|
||||
resolution,
|
||||
fps
|
||||
}
|
||||
}
|
||||
|
||||
const privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload = {
|
||||
...pick(options, [ 'isNewVideo' ]),
|
||||
|
||||
videoUUID: video.uuid
|
||||
}
|
||||
|
||||
const job = await this.createRunnerJob({
|
||||
type: 'vod-audio-merge-transcoding',
|
||||
jobUUID,
|
||||
payload,
|
||||
privatePayload,
|
||||
priority,
|
||||
dependsOnRunnerJob
|
||||
})
|
||||
|
||||
await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode')
|
||||
|
||||
return job
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async specificComplete (options: {
|
||||
runnerJob: MRunnerJob
|
||||
resultPayload: VODAudioMergeTranscodingSuccess
|
||||
}) {
|
||||
const { runnerJob, resultPayload } = options
|
||||
const privatePayload = runnerJob.privatePayload as RunnerJobVODWebVideoTranscodingPrivatePayload
|
||||
|
||||
const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags)
|
||||
if (!video) return
|
||||
|
||||
const videoFilePath = resultPayload.videoFile as string
|
||||
|
||||
// ffmpeg generated a new video file, so update the video duration
|
||||
// See https://trac.ffmpeg.org/ticket/5456
|
||||
video.duration = await getVideoStreamDuration(videoFilePath)
|
||||
await video.save()
|
||||
|
||||
// We can remove the old audio file
|
||||
const oldAudioFile = video.VideoFiles[0]
|
||||
await video.removeWebTorrentFile(oldAudioFile)
|
||||
await oldAudioFile.destroy()
|
||||
video.VideoFiles = []
|
||||
|
||||
await onVODWebVideoOrAudioMergeTranscodingJob({ video, videoFilePath, privatePayload })
|
||||
|
||||
logger.info(
|
||||
'Runner VOD audio merge transcoding job %s for %s ended.',
|
||||
runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid)
|
||||
)
|
||||
}
|
||||
}
|
@ -0,0 +1,114 @@
|
||||
import { move } from 'fs-extra'
|
||||
import { dirname, join } from 'path'
|
||||
import { logger } from '@server/helpers/logger'
|
||||
import { renameVideoFileInPlaylist } from '@server/lib/hls'
|
||||
import { getHlsResolutionPlaylistFilename } from '@server/lib/paths'
|
||||
import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding'
|
||||
import { onHLSVideoFileTranscoding } from '@server/lib/transcoding/hls-transcoding'
|
||||
import { buildNewFile, removeAllWebTorrentFiles } from '@server/lib/video-file'
|
||||
import { VideoJobInfoModel } from '@server/models/video/video-job-info'
|
||||
import { MVideo } from '@server/types/models'
|
||||
import { MRunnerJob } from '@server/types/models/runners'
|
||||
import { pick } from '@shared/core-utils'
|
||||
import { buildUUID } from '@shared/extra-utils'
|
||||
import {
|
||||
RunnerJobUpdatePayload,
|
||||
RunnerJobVODHLSTranscodingPayload,
|
||||
RunnerJobVODHLSTranscodingPrivatePayload,
|
||||
VODHLSTranscodingSuccess
|
||||
} from '@shared/models'
|
||||
import { generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls'
|
||||
import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler'
|
||||
import { loadTranscodingRunnerVideo } from './shared'
|
||||
|
||||
type CreateOptions = {
|
||||
video: MVideo
|
||||
isNewVideo: boolean
|
||||
deleteWebVideoFiles: boolean
|
||||
resolution: number
|
||||
fps: number
|
||||
priority: number
|
||||
dependsOnRunnerJob?: MRunnerJob
|
||||
}
|
||||
|
||||
// eslint-disable-next-line max-len
|
||||
export class VODHLSTranscodingJobHandler extends AbstractVODTranscodingJobHandler<CreateOptions, RunnerJobUpdatePayload, VODHLSTranscodingSuccess> {
|
||||
|
||||
async create (options: CreateOptions) {
|
||||
const { video, resolution, fps, dependsOnRunnerJob, priority } = options
|
||||
|
||||
const jobUUID = buildUUID()
|
||||
|
||||
const payload: RunnerJobVODHLSTranscodingPayload = {
|
||||
input: {
|
||||
videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid)
|
||||
},
|
||||
output: {
|
||||
resolution,
|
||||
fps
|
||||
}
|
||||
}
|
||||
|
||||
const privatePayload: RunnerJobVODHLSTranscodingPrivatePayload = {
|
||||
...pick(options, [ 'isNewVideo', 'deleteWebVideoFiles' ]),
|
||||
|
||||
videoUUID: video.uuid
|
||||
}
|
||||
|
||||
const job = await this.createRunnerJob({
|
||||
type: 'vod-hls-transcoding',
|
||||
jobUUID,
|
||||
payload,
|
||||
privatePayload,
|
||||
priority,
|
||||
dependsOnRunnerJob
|
||||
})
|
||||
|
||||
await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode')
|
||||
|
||||
return job
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async specificComplete (options: {
|
||||
runnerJob: MRunnerJob
|
||||
resultPayload: VODHLSTranscodingSuccess
|
||||
}) {
|
||||
const { runnerJob, resultPayload } = options
|
||||
const privatePayload = runnerJob.privatePayload as RunnerJobVODHLSTranscodingPrivatePayload
|
||||
|
||||
const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags)
|
||||
if (!video) return
|
||||
|
||||
const videoFilePath = resultPayload.videoFile as string
|
||||
const resolutionPlaylistFilePath = resultPayload.resolutionPlaylistFile as string
|
||||
|
||||
const videoFile = await buildNewFile({ path: videoFilePath, mode: 'hls' })
|
||||
const newVideoFilePath = join(dirname(videoFilePath), videoFile.filename)
|
||||
await move(videoFilePath, newVideoFilePath)
|
||||
|
||||
const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFile.filename)
|
||||
const newResolutionPlaylistFilePath = join(dirname(resolutionPlaylistFilePath), resolutionPlaylistFilename)
|
||||
await move(resolutionPlaylistFilePath, newResolutionPlaylistFilePath)
|
||||
|
||||
await renameVideoFileInPlaylist(newResolutionPlaylistFilePath, videoFile.filename)
|
||||
|
||||
await onHLSVideoFileTranscoding({
|
||||
video,
|
||||
videoFile,
|
||||
m3u8OutputPath: newResolutionPlaylistFilePath,
|
||||
videoOutputPath: newVideoFilePath
|
||||
})
|
||||
|
||||
await onTranscodingEnded({ isNewVideo: privatePayload.isNewVideo, moveVideoToNextState: true, video })
|
||||
|
||||
if (privatePayload.deleteWebVideoFiles === true) {
|
||||
logger.info('Removing web video files of %s now we have a HLS version of it.', video.uuid, this.lTags(video.uuid))
|
||||
|
||||
await removeAllWebTorrentFiles(video)
|
||||
}
|
||||
|
||||
logger.info('Runner VOD HLS job %s for %s ended.', runnerJob.uuid, video.uuid, this.lTags(runnerJob.uuid, video.uuid))
|
||||
}
|
||||
}
|
@ -0,0 +1,84 @@
|
||||
import { pick } from 'lodash'
|
||||
import { logger } from '@server/helpers/logger'
|
||||
import { VideoJobInfoModel } from '@server/models/video/video-job-info'
|
||||
import { MVideo } from '@server/types/models'
|
||||
import { MRunnerJob } from '@server/types/models/runners'
|
||||
import { buildUUID } from '@shared/extra-utils'
|
||||
import {
|
||||
RunnerJobUpdatePayload,
|
||||
RunnerJobVODWebVideoTranscodingPayload,
|
||||
RunnerJobVODWebVideoTranscodingPrivatePayload,
|
||||
VODWebVideoTranscodingSuccess
|
||||
} from '@shared/models'
|
||||
import { generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls'
|
||||
import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler'
|
||||
import { loadTranscodingRunnerVideo, onVODWebVideoOrAudioMergeTranscodingJob } from './shared'
|
||||
|
||||
type CreateOptions = {
|
||||
video: MVideo
|
||||
isNewVideo: boolean
|
||||
resolution: number
|
||||
fps: number
|
||||
priority: number
|
||||
dependsOnRunnerJob?: MRunnerJob
|
||||
}
|
||||
|
||||
// eslint-disable-next-line max-len
|
||||
export class VODWebVideoTranscodingJobHandler extends AbstractVODTranscodingJobHandler<CreateOptions, RunnerJobUpdatePayload, VODWebVideoTranscodingSuccess> {
|
||||
|
||||
async create (options: CreateOptions) {
|
||||
const { video, resolution, fps, priority, dependsOnRunnerJob } = options
|
||||
|
||||
const jobUUID = buildUUID()
|
||||
const payload: RunnerJobVODWebVideoTranscodingPayload = {
|
||||
input: {
|
||||
videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid)
|
||||
},
|
||||
output: {
|
||||
resolution,
|
||||
fps
|
||||
}
|
||||
}
|
||||
|
||||
const privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload = {
|
||||
...pick(options, [ 'isNewVideo' ]),
|
||||
|
||||
videoUUID: video.uuid
|
||||
}
|
||||
|
||||
const job = await this.createRunnerJob({
|
||||
type: 'vod-web-video-transcoding',
|
||||
jobUUID,
|
||||
payload,
|
||||
privatePayload,
|
||||
dependsOnRunnerJob,
|
||||
priority
|
||||
})
|
||||
|
||||
await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode')
|
||||
|
||||
return job
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async specificComplete (options: {
|
||||
runnerJob: MRunnerJob
|
||||
resultPayload: VODWebVideoTranscodingSuccess
|
||||
}) {
|
||||
const { runnerJob, resultPayload } = options
|
||||
const privatePayload = runnerJob.privatePayload as RunnerJobVODWebVideoTranscodingPrivatePayload
|
||||
|
||||
const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags)
|
||||
if (!video) return
|
||||
|
||||
const videoFilePath = resultPayload.videoFile as string
|
||||
|
||||
await onVODWebVideoOrAudioMergeTranscodingJob({ video, videoFilePath, privatePayload })
|
||||
|
||||
logger.info(
|
||||
'Runner VOD web video transcoding job %s for %s ended.',
|
||||
runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid)
|
||||
)
|
||||
}
|
||||
}
|
9
server/lib/runners/runner-urls.ts
Normal file
9
server/lib/runners/runner-urls.ts
Normal file
@ -0,0 +1,9 @@
|
||||
import { WEBSERVER } from '@server/initializers/constants'
|
||||
|
||||
export function generateRunnerTranscodingVideoInputFileUrl (jobUUID: string, videoUUID: string) {
|
||||
return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/max-quality'
|
||||
}
|
||||
|
||||
export function generateRunnerTranscodingVideoPreviewFileUrl (jobUUID: string, videoUUID: string) {
|
||||
return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/previews/max-quality'
|
||||
}
|
36
server/lib/runners/runner.ts
Normal file
36
server/lib/runners/runner.ts
Normal file
@ -0,0 +1,36 @@
|
||||
import express from 'express'
|
||||
import { retryTransactionWrapper } from '@server/helpers/database-utils'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger'
|
||||
import { sequelizeTypescript } from '@server/initializers/database'
|
||||
import { MRunner } from '@server/types/models/runners'
|
||||
|
||||
const lTags = loggerTagsFactory('runner')
|
||||
|
||||
const updatingRunner = new Set<number>()
|
||||
|
||||
function updateLastRunnerContact (req: express.Request, runner: MRunner) {
|
||||
const now = new Date()
|
||||
|
||||
// Don't update last runner contact too often
|
||||
if (now.getTime() - runner.lastContact.getTime() < 2000) return
|
||||
if (updatingRunner.has(runner.id)) return
|
||||
|
||||
updatingRunner.add(runner.id)
|
||||
|
||||
runner.lastContact = now
|
||||
runner.ip = req.ip
|
||||
|
||||
logger.debug('Updating last runner contact for %s', runner.name, lTags(runner.name))
|
||||
|
||||
retryTransactionWrapper(() => {
|
||||
return sequelizeTypescript.transaction(async transaction => {
|
||||
return runner.save({ transaction })
|
||||
})
|
||||
})
|
||||
.catch(err => logger.error('Cannot update last runner contact for %s', runner.name, { err, ...lTags(runner.name) }))
|
||||
.finally(() => updatingRunner.delete(runner.id))
|
||||
}
|
||||
|
||||
export {
|
||||
updateLastRunnerContact
|
||||
}
|
42
server/lib/schedulers/runner-job-watch-dog-scheduler.ts
Normal file
42
server/lib/schedulers/runner-job-watch-dog-scheduler.ts
Normal file
@ -0,0 +1,42 @@
|
||||
import { CONFIG } from '@server/initializers/config'
|
||||
import { RunnerJobModel } from '@server/models/runner/runner-job'
|
||||
import { logger, loggerTagsFactory } from '../../helpers/logger'
|
||||
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
|
||||
import { getRunnerJobHandlerClass } from '../runners'
|
||||
import { AbstractScheduler } from './abstract-scheduler'
|
||||
|
||||
const lTags = loggerTagsFactory('runner')
|
||||
|
||||
export class RunnerJobWatchDogScheduler extends AbstractScheduler {
|
||||
|
||||
private static instance: AbstractScheduler
|
||||
|
||||
protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.RUNNER_JOB_WATCH_DOG
|
||||
|
||||
private constructor () {
|
||||
super()
|
||||
}
|
||||
|
||||
protected async internalExecute () {
|
||||
const vodStalledJobs = await RunnerJobModel.listStalledJobs({
|
||||
staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.VOD,
|
||||
types: [ 'vod-audio-merge-transcoding', 'vod-hls-transcoding', 'vod-web-video-transcoding' ]
|
||||
})
|
||||
|
||||
const liveStalledJobs = await RunnerJobModel.listStalledJobs({
|
||||
staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.LIVE,
|
||||
types: [ 'live-rtmp-hls-transcoding' ]
|
||||
})
|
||||
|
||||
for (const stalled of [ ...vodStalledJobs, ...liveStalledJobs ]) {
|
||||
logger.info('Abort stalled runner job %s (%s)', stalled.uuid, stalled.type, lTags(stalled.uuid, stalled.type))
|
||||
|
||||
const Handler = getRunnerJobHandlerClass(stalled)
|
||||
await new Handler().abort({ runnerJob: stalled })
|
||||
}
|
||||
}
|
||||
|
||||
static get Instance () {
|
||||
return this.instance || (this.instance = new this())
|
||||
}
|
||||
}
|
@ -126,11 +126,14 @@ class ServerConfigManager {
|
||||
serverVersion: PEERTUBE_VERSION,
|
||||
serverCommit: this.serverCommit,
|
||||
transcoding: {
|
||||
remoteRunners: {
|
||||
enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED
|
||||
},
|
||||
hls: {
|
||||
enabled: CONFIG.TRANSCODING.HLS.ENABLED
|
||||
enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.HLS.ENABLED
|
||||
},
|
||||
webtorrent: {
|
||||
enabled: CONFIG.TRANSCODING.WEBTORRENT.ENABLED
|
||||
enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.WEBTORRENT.ENABLED
|
||||
},
|
||||
enabledResolutions: this.getEnabledResolutions('vod'),
|
||||
profile: CONFIG.TRANSCODING.PROFILE,
|
||||
@ -150,6 +153,9 @@ class ServerConfigManager {
|
||||
|
||||
transcoding: {
|
||||
enabled: CONFIG.LIVE.TRANSCODING.ENABLED,
|
||||
remoteRunners: {
|
||||
enabled: CONFIG.LIVE.TRANSCODING.ENABLED && CONFIG.LIVE.TRANSCODING.REMOTE_RUNNERS.ENABLED
|
||||
},
|
||||
enabledResolutions: this.getEnabledResolutions('live'),
|
||||
profile: CONFIG.LIVE.TRANSCODING.PROFILE,
|
||||
availableProfiles: VideoTranscodingProfilesManager.Instance.getAvailableProfiles('live')
|
||||
|
36
server/lib/transcoding/create-transcoding-job.ts
Normal file
36
server/lib/transcoding/create-transcoding-job.ts
Normal file
@ -0,0 +1,36 @@
|
||||
import { CONFIG } from '@server/initializers/config'
|
||||
import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models'
|
||||
import { TranscodingJobQueueBuilder, TranscodingRunnerJobBuilder } from './shared'
|
||||
|
||||
export function createOptimizeOrMergeAudioJobs (options: {
|
||||
video: MVideoFullLight
|
||||
videoFile: MVideoFile
|
||||
isNewVideo: boolean
|
||||
user: MUserId
|
||||
}) {
|
||||
return getJobBuilder().createOptimizeOrMergeAudioJobs(options)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function createTranscodingJobs (options: {
|
||||
transcodingType: 'hls' | 'webtorrent'
|
||||
video: MVideoFullLight
|
||||
resolutions: number[]
|
||||
isNewVideo: boolean
|
||||
user: MUserId
|
||||
}) {
|
||||
return getJobBuilder().createTranscodingJobs(options)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Private
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function getJobBuilder () {
|
||||
if (CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED === true) {
|
||||
return new TranscodingRunnerJobBuilder()
|
||||
}
|
||||
|
||||
return new TranscodingJobQueueBuilder()
|
||||
}
|
@ -1,15 +1,9 @@
|
||||
|
||||
import { logger } from '@server/helpers/logger'
|
||||
import { getAverageBitrate, getMinLimitBitrate } from '@shared/core-utils'
|
||||
import { AvailableEncoders, EncoderOptionsBuilder, EncoderOptionsBuilderParams, VideoResolution } from '../../../shared/models/videos'
|
||||
import {
|
||||
buildStreamSuffix,
|
||||
canDoQuickAudioTranscode,
|
||||
ffprobePromise,
|
||||
getAudioStream,
|
||||
getMaxAudioBitrate,
|
||||
resetSupportedEncoders
|
||||
} from '../../helpers/ffmpeg'
|
||||
import { buildStreamSuffix, FFmpegCommandWrapper, ffprobePromise, getAudioStream, getMaxAudioBitrate } from '@shared/ffmpeg'
|
||||
import { AvailableEncoders, EncoderOptionsBuilder, EncoderOptionsBuilderParams, VideoResolution } from '@shared/models'
|
||||
import { canDoQuickAudioTranscode } from './transcoding-quick-transcode'
|
||||
|
||||
/**
|
||||
*
|
||||
@ -184,14 +178,14 @@ class VideoTranscodingProfilesManager {
|
||||
addEncoderPriority (type: 'vod' | 'live', streamType: 'audio' | 'video', encoder: string, priority: number) {
|
||||
this.encodersPriorities[type][streamType].push({ name: encoder, priority })
|
||||
|
||||
resetSupportedEncoders()
|
||||
FFmpegCommandWrapper.resetSupportedEncoders()
|
||||
}
|
||||
|
||||
removeEncoderPriority (type: 'vod' | 'live', streamType: 'audio' | 'video', encoder: string, priority: number) {
|
||||
this.encodersPriorities[type][streamType] = this.encodersPriorities[type][streamType]
|
||||
.filter(o => o.name !== encoder && o.priority !== priority)
|
||||
|
||||
resetSupportedEncoders()
|
||||
FFmpegCommandWrapper.resetSupportedEncoders()
|
||||
}
|
||||
|
||||
private getEncodersByPriority (type: 'vod' | 'live', streamType: 'audio' | 'video') {
|
||||
|
18
server/lib/transcoding/ended-transcoding.ts
Normal file
18
server/lib/transcoding/ended-transcoding.ts
Normal file
@ -0,0 +1,18 @@
|
||||
import { retryTransactionWrapper } from '@server/helpers/database-utils'
|
||||
import { VideoJobInfoModel } from '@server/models/video/video-job-info'
|
||||
import { MVideo } from '@server/types/models'
|
||||
import { moveToNextState } from '../video-state'
|
||||
|
||||
export async function onTranscodingEnded (options: {
|
||||
video: MVideo
|
||||
isNewVideo: boolean
|
||||
moveVideoToNextState: boolean
|
||||
}) {
|
||||
const { video, isNewVideo, moveVideoToNextState } = options
|
||||
|
||||
await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
|
||||
|
||||
if (moveVideoToNextState) {
|
||||
await retryTransactionWrapper(moveToNextState, { video, isNewVideo })
|
||||
}
|
||||
}
|
181
server/lib/transcoding/hls-transcoding.ts
Normal file
181
server/lib/transcoding/hls-transcoding.ts
Normal file
@ -0,0 +1,181 @@
|
||||
import { MutexInterface } from 'async-mutex'
|
||||
import { Job } from 'bullmq'
|
||||
import { ensureDir, move, stat } from 'fs-extra'
|
||||
import { basename, extname as extnameUtil, join } from 'path'
|
||||
import { retryTransactionWrapper } from '@server/helpers/database-utils'
|
||||
import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
|
||||
import { sequelizeTypescript } from '@server/initializers/database'
|
||||
import { MVideo, MVideoFile } from '@server/types/models'
|
||||
import { pick } from '@shared/core-utils'
|
||||
import { getVideoStreamDuration, getVideoStreamFPS } from '@shared/ffmpeg'
|
||||
import { VideoResolution } from '@shared/models'
|
||||
import { CONFIG } from '../../initializers/config'
|
||||
import { VideoFileModel } from '../../models/video/video-file'
|
||||
import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
|
||||
import { updatePlaylistAfterFileChange } from '../hls'
|
||||
import { generateHLSVideoFilename, getHlsResolutionPlaylistFilename } from '../paths'
|
||||
import { buildFileMetadata } from '../video-file'
|
||||
import { VideoPathManager } from '../video-path-manager'
|
||||
import { buildFFmpegVOD } from './shared'
|
||||
|
||||
// Concat TS segments from a live video to a fragmented mp4 HLS playlist
|
||||
export async function generateHlsPlaylistResolutionFromTS (options: {
|
||||
video: MVideo
|
||||
concatenatedTsFilePath: string
|
||||
resolution: VideoResolution
|
||||
fps: number
|
||||
isAAC: boolean
|
||||
inputFileMutexReleaser: MutexInterface.Releaser
|
||||
}) {
|
||||
return generateHlsPlaylistCommon({
|
||||
type: 'hls-from-ts' as 'hls-from-ts',
|
||||
inputPath: options.concatenatedTsFilePath,
|
||||
|
||||
...pick(options, [ 'video', 'resolution', 'fps', 'inputFileMutexReleaser', 'isAAC' ])
|
||||
})
|
||||
}
|
||||
|
||||
// Generate an HLS playlist from an input file, and update the master playlist
|
||||
export function generateHlsPlaylistResolution (options: {
|
||||
video: MVideo
|
||||
videoInputPath: string
|
||||
resolution: VideoResolution
|
||||
fps: number
|
||||
copyCodecs: boolean
|
||||
inputFileMutexReleaser: MutexInterface.Releaser
|
||||
job?: Job
|
||||
}) {
|
||||
return generateHlsPlaylistCommon({
|
||||
type: 'hls' as 'hls',
|
||||
inputPath: options.videoInputPath,
|
||||
|
||||
...pick(options, [ 'video', 'resolution', 'fps', 'copyCodecs', 'inputFileMutexReleaser', 'job' ])
|
||||
})
|
||||
}
|
||||
|
||||
export async function onHLSVideoFileTranscoding (options: {
|
||||
video: MVideo
|
||||
videoFile: MVideoFile
|
||||
videoOutputPath: string
|
||||
m3u8OutputPath: string
|
||||
}) {
|
||||
const { video, videoFile, videoOutputPath, m3u8OutputPath } = options
|
||||
|
||||
// Create or update the playlist
|
||||
const playlist = await retryTransactionWrapper(() => {
|
||||
return sequelizeTypescript.transaction(async transaction => {
|
||||
return VideoStreamingPlaylistModel.loadOrGenerate(video, transaction)
|
||||
})
|
||||
})
|
||||
videoFile.videoStreamingPlaylistId = playlist.id
|
||||
|
||||
const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
|
||||
|
||||
try {
|
||||
// VOD transcoding is a long task, refresh video attributes
|
||||
await video.reload()
|
||||
|
||||
const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, videoFile)
|
||||
await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video))
|
||||
|
||||
// Move playlist file
|
||||
const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, basename(m3u8OutputPath))
|
||||
await move(m3u8OutputPath, resolutionPlaylistPath, { overwrite: true })
|
||||
// Move video file
|
||||
await move(videoOutputPath, videoFilePath, { overwrite: true })
|
||||
|
||||
// Update video duration if it was not set (in case of a live for example)
|
||||
if (!video.duration) {
|
||||
video.duration = await getVideoStreamDuration(videoFilePath)
|
||||
await video.save()
|
||||
}
|
||||
|
||||
const stats = await stat(videoFilePath)
|
||||
|
||||
videoFile.size = stats.size
|
||||
videoFile.fps = await getVideoStreamFPS(videoFilePath)
|
||||
videoFile.metadata = await buildFileMetadata(videoFilePath)
|
||||
|
||||
await createTorrentAndSetInfoHash(playlist, videoFile)
|
||||
|
||||
const oldFile = await VideoFileModel.loadHLSFile({
|
||||
playlistId: playlist.id,
|
||||
fps: videoFile.fps,
|
||||
resolution: videoFile.resolution
|
||||
})
|
||||
|
||||
if (oldFile) {
|
||||
await video.removeStreamingPlaylistVideoFile(playlist, oldFile)
|
||||
await oldFile.destroy()
|
||||
}
|
||||
|
||||
const savedVideoFile = await VideoFileModel.customUpsert(videoFile, 'streaming-playlist', undefined)
|
||||
|
||||
await updatePlaylistAfterFileChange(video, playlist)
|
||||
|
||||
return { resolutionPlaylistPath, videoFile: savedVideoFile }
|
||||
} finally {
|
||||
mutexReleaser()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function generateHlsPlaylistCommon (options: {
|
||||
type: 'hls' | 'hls-from-ts'
|
||||
video: MVideo
|
||||
inputPath: string
|
||||
|
||||
resolution: VideoResolution
|
||||
fps: number
|
||||
|
||||
inputFileMutexReleaser: MutexInterface.Releaser
|
||||
|
||||
copyCodecs?: boolean
|
||||
isAAC?: boolean
|
||||
|
||||
job?: Job
|
||||
}) {
|
||||
const { type, video, inputPath, resolution, fps, copyCodecs, isAAC, job, inputFileMutexReleaser } = options
|
||||
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
|
||||
|
||||
const videoTranscodedBasePath = join(transcodeDirectory, type)
|
||||
await ensureDir(videoTranscodedBasePath)
|
||||
|
||||
const videoFilename = generateHLSVideoFilename(resolution)
|
||||
const videoOutputPath = join(videoTranscodedBasePath, videoFilename)
|
||||
|
||||
const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFilename)
|
||||
const m3u8OutputPath = join(videoTranscodedBasePath, resolutionPlaylistFilename)
|
||||
|
||||
const transcodeOptions = {
|
||||
type,
|
||||
|
||||
inputPath,
|
||||
outputPath: m3u8OutputPath,
|
||||
|
||||
resolution,
|
||||
fps,
|
||||
copyCodecs,
|
||||
|
||||
isAAC,
|
||||
|
||||
inputFileMutexReleaser,
|
||||
|
||||
hlsPlaylist: {
|
||||
videoFilename
|
||||
}
|
||||
}
|
||||
|
||||
await buildFFmpegVOD(job).transcode(transcodeOptions)
|
||||
|
||||
const newVideoFile = new VideoFileModel({
|
||||
resolution,
|
||||
extname: extnameUtil(videoFilename),
|
||||
size: 0,
|
||||
filename: videoFilename,
|
||||
fps: -1
|
||||
})
|
||||
|
||||
await onHLSVideoFileTranscoding({ video, videoFile: newVideoFile, videoOutputPath, m3u8OutputPath })
|
||||
}
|
18
server/lib/transcoding/shared/ffmpeg-builder.ts
Normal file
18
server/lib/transcoding/shared/ffmpeg-builder.ts
Normal file
@ -0,0 +1,18 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg'
|
||||
import { logger } from '@server/helpers/logger'
|
||||
import { FFmpegVOD } from '@shared/ffmpeg'
|
||||
import { VideoTranscodingProfilesManager } from '../default-transcoding-profiles'
|
||||
|
||||
export function buildFFmpegVOD (job?: Job) {
|
||||
return new FFmpegVOD({
|
||||
...getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders()),
|
||||
|
||||
updateJobProgress: progress => {
|
||||
if (!job) return
|
||||
|
||||
job.updateProgress(progress)
|
||||
.catch(err => logger.error('Cannot update ffmpeg job progress', { err }))
|
||||
}
|
||||
})
|
||||
}
|
2
server/lib/transcoding/shared/index.ts
Normal file
2
server/lib/transcoding/shared/index.ts
Normal file
@ -0,0 +1,2 @@
|
||||
export * from './job-builders'
|
||||
export * from './ffmpeg-builder'
|
@ -0,0 +1,38 @@
|
||||
|
||||
import { JOB_PRIORITY } from '@server/initializers/constants'
|
||||
import { VideoModel } from '@server/models/video/video'
|
||||
import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models'
|
||||
|
||||
export abstract class AbstractJobBuilder {
|
||||
|
||||
abstract createOptimizeOrMergeAudioJobs (options: {
|
||||
video: MVideoFullLight
|
||||
videoFile: MVideoFile
|
||||
isNewVideo: boolean
|
||||
user: MUserId
|
||||
}): Promise<any>
|
||||
|
||||
abstract createTranscodingJobs (options: {
|
||||
transcodingType: 'hls' | 'webtorrent'
|
||||
video: MVideoFullLight
|
||||
resolutions: number[]
|
||||
isNewVideo: boolean
|
||||
user: MUserId | null
|
||||
}): Promise<any>
|
||||
|
||||
protected async getTranscodingJobPriority (options: {
|
||||
user: MUserId
|
||||
fallback: number
|
||||
}) {
|
||||
const { user, fallback } = options
|
||||
|
||||
if (!user) return fallback
|
||||
|
||||
const now = new Date()
|
||||
const lastWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7)
|
||||
|
||||
const videoUploadedByUser = await VideoModel.countVideosUploadedByUserSince(user.id, lastWeek)
|
||||
|
||||
return JOB_PRIORITY.TRANSCODING + videoUploadedByUser
|
||||
}
|
||||
}
|
2
server/lib/transcoding/shared/job-builders/index.ts
Normal file
2
server/lib/transcoding/shared/job-builders/index.ts
Normal file
@ -0,0 +1,2 @@
|
||||
export * from './transcoding-job-queue-builder'
|
||||
export * from './transcoding-runner-job-builder'
|
@ -0,0 +1,308 @@
|
||||
import Bluebird from 'bluebird'
|
||||
import { computeOutputFPS } from '@server/helpers/ffmpeg'
|
||||
import { logger } from '@server/helpers/logger'
|
||||
import { CONFIG } from '@server/initializers/config'
|
||||
import { DEFAULT_AUDIO_RESOLUTION, VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
|
||||
import { CreateJobArgument, JobQueue } from '@server/lib/job-queue'
|
||||
import { Hooks } from '@server/lib/plugins/hooks'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager'
|
||||
import { VideoJobInfoModel } from '@server/models/video/video-job-info'
|
||||
import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models'
|
||||
import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@shared/ffmpeg'
|
||||
import {
|
||||
HLSTranscodingPayload,
|
||||
MergeAudioTranscodingPayload,
|
||||
NewWebTorrentResolutionTranscodingPayload,
|
||||
OptimizeTranscodingPayload,
|
||||
VideoTranscodingPayload
|
||||
} from '@shared/models'
|
||||
import { canDoQuickTranscode } from '../../transcoding-quick-transcode'
|
||||
import { computeResolutionsToTranscode } from '../../transcoding-resolutions'
|
||||
import { AbstractJobBuilder } from './abstract-job-builder'
|
||||
|
||||
export class TranscodingJobQueueBuilder extends AbstractJobBuilder {
|
||||
|
||||
async createOptimizeOrMergeAudioJobs (options: {
|
||||
video: MVideoFullLight
|
||||
videoFile: MVideoFile
|
||||
isNewVideo: boolean
|
||||
user: MUserId
|
||||
}) {
|
||||
const { video, videoFile, isNewVideo, user } = options
|
||||
|
||||
let mergeOrOptimizePayload: MergeAudioTranscodingPayload | OptimizeTranscodingPayload
|
||||
let nextTranscodingSequentialJobPayloads: (NewWebTorrentResolutionTranscodingPayload | HLSTranscodingPayload)[][] = []
|
||||
|
||||
const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
|
||||
|
||||
try {
|
||||
await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), async videoFilePath => {
|
||||
const probe = await ffprobePromise(videoFilePath)
|
||||
|
||||
const { resolution } = await getVideoStreamDimensionsInfo(videoFilePath, probe)
|
||||
const hasAudio = await hasAudioStream(videoFilePath, probe)
|
||||
const quickTranscode = await canDoQuickTranscode(videoFilePath, probe)
|
||||
const inputFPS = videoFile.isAudio()
|
||||
? VIDEO_TRANSCODING_FPS.AUDIO_MERGE // The first transcoding job will transcode to this FPS value
|
||||
: await getVideoStreamFPS(videoFilePath, probe)
|
||||
|
||||
const maxResolution = await isAudioFile(videoFilePath, probe)
|
||||
? DEFAULT_AUDIO_RESOLUTION
|
||||
: resolution
|
||||
|
||||
if (CONFIG.TRANSCODING.HLS.ENABLED === true) {
|
||||
nextTranscodingSequentialJobPayloads.push([
|
||||
this.buildHLSJobPayload({
|
||||
deleteWebTorrentFiles: CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false,
|
||||
|
||||
// We had some issues with a web video quick transcoded while producing a HLS version of it
|
||||
copyCodecs: !quickTranscode,
|
||||
|
||||
resolution: maxResolution,
|
||||
fps: computeOutputFPS({ inputFPS, resolution: maxResolution }),
|
||||
videoUUID: video.uuid,
|
||||
isNewVideo
|
||||
})
|
||||
])
|
||||
}
|
||||
|
||||
const lowerResolutionJobPayloads = await this.buildLowerResolutionJobPayloads({
|
||||
video,
|
||||
inputVideoResolution: maxResolution,
|
||||
inputVideoFPS: inputFPS,
|
||||
hasAudio,
|
||||
isNewVideo
|
||||
})
|
||||
|
||||
nextTranscodingSequentialJobPayloads = [ ...nextTranscodingSequentialJobPayloads, ...lowerResolutionJobPayloads ]
|
||||
|
||||
mergeOrOptimizePayload = videoFile.isAudio()
|
||||
? this.buildMergeAudioPayload({ videoUUID: video.uuid, isNewVideo })
|
||||
: this.buildOptimizePayload({ videoUUID: video.uuid, isNewVideo, quickTranscode })
|
||||
})
|
||||
} finally {
|
||||
mutexReleaser()
|
||||
}
|
||||
|
||||
const nextTranscodingSequentialJobs = await Bluebird.mapSeries(nextTranscodingSequentialJobPayloads, payloads => {
|
||||
return Bluebird.mapSeries(payloads, payload => {
|
||||
return this.buildTranscodingJob({ payload, user })
|
||||
})
|
||||
})
|
||||
|
||||
const transcodingJobBuilderJob: CreateJobArgument = {
|
||||
type: 'transcoding-job-builder',
|
||||
payload: {
|
||||
videoUUID: video.uuid,
|
||||
sequentialJobs: nextTranscodingSequentialJobs
|
||||
}
|
||||
}
|
||||
|
||||
const mergeOrOptimizeJob = await this.buildTranscodingJob({ payload: mergeOrOptimizePayload, user })
|
||||
|
||||
return JobQueue.Instance.createSequentialJobFlow(...[ mergeOrOptimizeJob, transcodingJobBuilderJob ])
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async createTranscodingJobs (options: {
|
||||
transcodingType: 'hls' | 'webtorrent'
|
||||
video: MVideoFullLight
|
||||
resolutions: number[]
|
||||
isNewVideo: boolean
|
||||
user: MUserId | null
|
||||
}) {
|
||||
const { video, transcodingType, resolutions, isNewVideo } = options
|
||||
|
||||
const maxResolution = Math.max(...resolutions)
|
||||
const childrenResolutions = resolutions.filter(r => r !== maxResolution)
|
||||
|
||||
logger.info('Manually creating transcoding jobs for %s.', transcodingType, { childrenResolutions, maxResolution })
|
||||
|
||||
const { fps: inputFPS } = await video.probeMaxQualityFile()
|
||||
|
||||
const children = childrenResolutions.map(resolution => {
|
||||
const fps = computeOutputFPS({ inputFPS, resolution })
|
||||
|
||||
if (transcodingType === 'hls') {
|
||||
return this.buildHLSJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo })
|
||||
}
|
||||
|
||||
if (transcodingType === 'webtorrent') {
|
||||
return this.buildWebTorrentJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo })
|
||||
}
|
||||
|
||||
throw new Error('Unknown transcoding type')
|
||||
})
|
||||
|
||||
const fps = computeOutputFPS({ inputFPS, resolution: maxResolution })
|
||||
|
||||
const parent = transcodingType === 'hls'
|
||||
? this.buildHLSJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo })
|
||||
: this.buildWebTorrentJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo })
|
||||
|
||||
// Process the last resolution after the other ones to prevent concurrency issue
|
||||
// Because low resolutions use the biggest one as ffmpeg input
|
||||
await this.createTranscodingJobsWithChildren({ videoUUID: video.uuid, parent, children, user: null })
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
private async createTranscodingJobsWithChildren (options: {
|
||||
videoUUID: string
|
||||
parent: (HLSTranscodingPayload | NewWebTorrentResolutionTranscodingPayload)
|
||||
children: (HLSTranscodingPayload | NewWebTorrentResolutionTranscodingPayload)[]
|
||||
user: MUserId | null
|
||||
}) {
|
||||
const { videoUUID, parent, children, user } = options
|
||||
|
||||
const parentJob = await this.buildTranscodingJob({ payload: parent, user })
|
||||
const childrenJobs = await Bluebird.mapSeries(children, c => this.buildTranscodingJob({ payload: c, user }))
|
||||
|
||||
await JobQueue.Instance.createJobWithChildren(parentJob, childrenJobs)
|
||||
|
||||
await VideoJobInfoModel.increaseOrCreate(videoUUID, 'pendingTranscode', 1 + children.length)
|
||||
}
|
||||
|
||||
private async buildTranscodingJob (options: {
|
||||
payload: VideoTranscodingPayload
|
||||
user: MUserId | null // null means we don't want priority
|
||||
}) {
|
||||
const { user, payload } = options
|
||||
|
||||
return {
|
||||
type: 'video-transcoding' as 'video-transcoding',
|
||||
priority: await this.getTranscodingJobPriority({ user, fallback: undefined }),
|
||||
payload
|
||||
}
|
||||
}
|
||||
|
||||
private async buildLowerResolutionJobPayloads (options: {
|
||||
video: MVideoWithFileThumbnail
|
||||
inputVideoResolution: number
|
||||
inputVideoFPS: number
|
||||
hasAudio: boolean
|
||||
isNewVideo: boolean
|
||||
}) {
|
||||
const { video, inputVideoResolution, inputVideoFPS, isNewVideo, hasAudio } = options
|
||||
|
||||
// Create transcoding jobs if there are enabled resolutions
|
||||
const resolutionsEnabled = await Hooks.wrapObject(
|
||||
computeResolutionsToTranscode({ input: inputVideoResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }),
|
||||
'filter:transcoding.auto.resolutions-to-transcode.result',
|
||||
options
|
||||
)
|
||||
|
||||
const sequentialPayloads: (NewWebTorrentResolutionTranscodingPayload | HLSTranscodingPayload)[][] = []
|
||||
|
||||
for (const resolution of resolutionsEnabled) {
|
||||
const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution })
|
||||
|
||||
if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED) {
|
||||
const payloads: (NewWebTorrentResolutionTranscodingPayload | HLSTranscodingPayload)[] = [
|
||||
this.buildWebTorrentJobPayload({
|
||||
videoUUID: video.uuid,
|
||||
resolution,
|
||||
fps,
|
||||
isNewVideo
|
||||
})
|
||||
]
|
||||
|
||||
// Create a subsequent job to create HLS resolution that will just copy web video codecs
|
||||
if (CONFIG.TRANSCODING.HLS.ENABLED) {
|
||||
payloads.push(
|
||||
this.buildHLSJobPayload({
|
||||
videoUUID: video.uuid,
|
||||
resolution,
|
||||
fps,
|
||||
isNewVideo,
|
||||
copyCodecs: true
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
sequentialPayloads.push(payloads)
|
||||
} else if (CONFIG.TRANSCODING.HLS.ENABLED) {
|
||||
sequentialPayloads.push([
|
||||
this.buildHLSJobPayload({
|
||||
videoUUID: video.uuid,
|
||||
resolution,
|
||||
fps,
|
||||
copyCodecs: false,
|
||||
isNewVideo
|
||||
})
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
return sequentialPayloads
|
||||
}
|
||||
|
||||
private buildHLSJobPayload (options: {
|
||||
videoUUID: string
|
||||
resolution: number
|
||||
fps: number
|
||||
isNewVideo: boolean
|
||||
deleteWebTorrentFiles?: boolean // default false
|
||||
copyCodecs?: boolean // default false
|
||||
}): HLSTranscodingPayload {
|
||||
const { videoUUID, resolution, fps, isNewVideo, deleteWebTorrentFiles = false, copyCodecs = false } = options
|
||||
|
||||
return {
|
||||
type: 'new-resolution-to-hls',
|
||||
videoUUID,
|
||||
resolution,
|
||||
fps,
|
||||
copyCodecs,
|
||||
isNewVideo,
|
||||
deleteWebTorrentFiles
|
||||
}
|
||||
}
|
||||
|
||||
private buildWebTorrentJobPayload (options: {
|
||||
videoUUID: string
|
||||
resolution: number
|
||||
fps: number
|
||||
isNewVideo: boolean
|
||||
}): NewWebTorrentResolutionTranscodingPayload {
|
||||
const { videoUUID, resolution, fps, isNewVideo } = options
|
||||
|
||||
return {
|
||||
type: 'new-resolution-to-webtorrent',
|
||||
videoUUID,
|
||||
isNewVideo,
|
||||
resolution,
|
||||
fps
|
||||
}
|
||||
}
|
||||
|
||||
private buildMergeAudioPayload (options: {
|
||||
videoUUID: string
|
||||
isNewVideo: boolean
|
||||
}): MergeAudioTranscodingPayload {
|
||||
const { videoUUID, isNewVideo } = options
|
||||
|
||||
return {
|
||||
type: 'merge-audio-to-webtorrent',
|
||||
resolution: DEFAULT_AUDIO_RESOLUTION,
|
||||
fps: VIDEO_TRANSCODING_FPS.AUDIO_MERGE,
|
||||
videoUUID,
|
||||
isNewVideo
|
||||
}
|
||||
}
|
||||
|
||||
private buildOptimizePayload (options: {
|
||||
videoUUID: string
|
||||
quickTranscode: boolean
|
||||
isNewVideo: boolean
|
||||
}): OptimizeTranscodingPayload {
|
||||
const { videoUUID, quickTranscode, isNewVideo } = options
|
||||
|
||||
return {
|
||||
type: 'optimize-to-webtorrent',
|
||||
videoUUID,
|
||||
isNewVideo,
|
||||
quickTranscode
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,189 @@
|
||||
import { computeOutputFPS } from '@server/helpers/ffmpeg'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger'
|
||||
import { CONFIG } from '@server/initializers/config'
|
||||
import { DEFAULT_AUDIO_RESOLUTION, VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
|
||||
import { Hooks } from '@server/lib/plugins/hooks'
|
||||
import { VODAudioMergeTranscodingJobHandler, VODHLSTranscodingJobHandler, VODWebVideoTranscodingJobHandler } from '@server/lib/runners'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager'
|
||||
import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models'
|
||||
import { MRunnerJob } from '@server/types/models/runners'
|
||||
import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@shared/ffmpeg'
|
||||
import { computeResolutionsToTranscode } from '../../transcoding-resolutions'
|
||||
import { AbstractJobBuilder } from './abstract-job-builder'
|
||||
|
||||
/**
|
||||
*
|
||||
* Class to build transcoding job in the local job queue
|
||||
*
|
||||
*/
|
||||
|
||||
const lTags = loggerTagsFactory('transcoding')
|
||||
|
||||
export class TranscodingRunnerJobBuilder extends AbstractJobBuilder {
|
||||
|
||||
async createOptimizeOrMergeAudioJobs (options: {
|
||||
video: MVideoFullLight
|
||||
videoFile: MVideoFile
|
||||
isNewVideo: boolean
|
||||
user: MUserId
|
||||
}) {
|
||||
const { video, videoFile, isNewVideo, user } = options
|
||||
|
||||
const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
|
||||
|
||||
try {
|
||||
await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), async videoFilePath => {
|
||||
const probe = await ffprobePromise(videoFilePath)
|
||||
|
||||
const { resolution } = await getVideoStreamDimensionsInfo(videoFilePath, probe)
|
||||
const hasAudio = await hasAudioStream(videoFilePath, probe)
|
||||
const inputFPS = videoFile.isAudio()
|
||||
? VIDEO_TRANSCODING_FPS.AUDIO_MERGE // The first transcoding job will transcode to this FPS value
|
||||
: await getVideoStreamFPS(videoFilePath, probe)
|
||||
|
||||
const maxResolution = await isAudioFile(videoFilePath, probe)
|
||||
? DEFAULT_AUDIO_RESOLUTION
|
||||
: resolution
|
||||
|
||||
const fps = computeOutputFPS({ inputFPS, resolution: maxResolution })
|
||||
const priority = await this.getTranscodingJobPriority({ user, fallback: 0 })
|
||||
|
||||
const mainRunnerJob = videoFile.isAudio()
|
||||
? await new VODAudioMergeTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority })
|
||||
: await new VODWebVideoTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority })
|
||||
|
||||
if (CONFIG.TRANSCODING.HLS.ENABLED === true) {
|
||||
await new VODHLSTranscodingJobHandler().create({
|
||||
video,
|
||||
deleteWebVideoFiles: CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false,
|
||||
resolution: maxResolution,
|
||||
fps,
|
||||
isNewVideo,
|
||||
dependsOnRunnerJob: mainRunnerJob,
|
||||
priority: await this.getTranscodingJobPriority({ user, fallback: 0 })
|
||||
})
|
||||
}
|
||||
|
||||
await this.buildLowerResolutionJobPayloads({
|
||||
video,
|
||||
inputVideoResolution: maxResolution,
|
||||
inputVideoFPS: inputFPS,
|
||||
hasAudio,
|
||||
isNewVideo,
|
||||
mainRunnerJob,
|
||||
user
|
||||
})
|
||||
})
|
||||
} finally {
|
||||
mutexReleaser()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async createTranscodingJobs (options: {
|
||||
transcodingType: 'hls' | 'webtorrent'
|
||||
video: MVideoFullLight
|
||||
resolutions: number[]
|
||||
isNewVideo: boolean
|
||||
user: MUserId | null
|
||||
}) {
|
||||
const { video, transcodingType, resolutions, isNewVideo, user } = options
|
||||
|
||||
const maxResolution = Math.max(...resolutions)
|
||||
const { fps: inputFPS } = await video.probeMaxQualityFile()
|
||||
const maxFPS = computeOutputFPS({ inputFPS, resolution: maxResolution })
|
||||
const priority = await this.getTranscodingJobPriority({ user, fallback: 0 })
|
||||
|
||||
const childrenResolutions = resolutions.filter(r => r !== maxResolution)
|
||||
|
||||
logger.info('Manually creating transcoding jobs for %s.', transcodingType, { childrenResolutions, maxResolution })
|
||||
|
||||
// Process the last resolution before the other ones to prevent concurrency issue
|
||||
// Because low resolutions use the biggest one as ffmpeg input
|
||||
const mainJob = transcodingType === 'hls'
|
||||
// eslint-disable-next-line max-len
|
||||
? await new VODHLSTranscodingJobHandler().create({ video, resolution: maxResolution, fps: maxFPS, isNewVideo, deleteWebVideoFiles: false, priority })
|
||||
: await new VODWebVideoTranscodingJobHandler().create({ video, resolution: maxResolution, fps: maxFPS, isNewVideo, priority })
|
||||
|
||||
for (const resolution of childrenResolutions) {
|
||||
const dependsOnRunnerJob = mainJob
|
||||
const fps = computeOutputFPS({ inputFPS, resolution: maxResolution })
|
||||
|
||||
if (transcodingType === 'hls') {
|
||||
await new VODHLSTranscodingJobHandler().create({
|
||||
video,
|
||||
resolution,
|
||||
fps,
|
||||
isNewVideo,
|
||||
deleteWebVideoFiles: false,
|
||||
dependsOnRunnerJob,
|
||||
priority: await this.getTranscodingJobPriority({ user, fallback: 0 })
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
if (transcodingType === 'webtorrent') {
|
||||
await new VODWebVideoTranscodingJobHandler().create({
|
||||
video,
|
||||
resolution,
|
||||
fps,
|
||||
isNewVideo,
|
||||
dependsOnRunnerJob,
|
||||
priority: await this.getTranscodingJobPriority({ user, fallback: 0 })
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
throw new Error('Unknown transcoding type')
|
||||
}
|
||||
}
|
||||
|
||||
private async buildLowerResolutionJobPayloads (options: {
|
||||
mainRunnerJob: MRunnerJob
|
||||
video: MVideoWithFileThumbnail
|
||||
inputVideoResolution: number
|
||||
inputVideoFPS: number
|
||||
hasAudio: boolean
|
||||
isNewVideo: boolean
|
||||
user: MUserId
|
||||
}) {
|
||||
const { video, inputVideoResolution, inputVideoFPS, isNewVideo, hasAudio, mainRunnerJob, user } = options
|
||||
|
||||
// Create transcoding jobs if there are enabled resolutions
|
||||
const resolutionsEnabled = await Hooks.wrapObject(
|
||||
computeResolutionsToTranscode({ input: inputVideoResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }),
|
||||
'filter:transcoding.auto.resolutions-to-transcode.result',
|
||||
options
|
||||
)
|
||||
|
||||
logger.debug('Lower resolutions build for %s.', video.uuid, { resolutionsEnabled, ...lTags(video.uuid) })
|
||||
|
||||
for (const resolution of resolutionsEnabled) {
|
||||
const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution })
|
||||
|
||||
if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED) {
|
||||
await new VODWebVideoTranscodingJobHandler().create({
|
||||
video,
|
||||
resolution,
|
||||
fps,
|
||||
isNewVideo,
|
||||
dependsOnRunnerJob: mainRunnerJob,
|
||||
priority: await this.getTranscodingJobPriority({ user, fallback: 0 })
|
||||
})
|
||||
}
|
||||
|
||||
if (CONFIG.TRANSCODING.HLS.ENABLED) {
|
||||
await new VODHLSTranscodingJobHandler().create({
|
||||
video,
|
||||
resolution,
|
||||
fps,
|
||||
isNewVideo,
|
||||
deleteWebVideoFiles: false,
|
||||
dependsOnRunnerJob: mainRunnerJob,
|
||||
priority: await this.getTranscodingJobPriority({ user, fallback: 0 })
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
61
server/lib/transcoding/transcoding-quick-transcode.ts
Normal file
61
server/lib/transcoding/transcoding-quick-transcode.ts
Normal file
@ -0,0 +1,61 @@
|
||||
import { FfprobeData } from 'fluent-ffmpeg'
|
||||
import { CONFIG } from '@server/initializers/config'
|
||||
import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
|
||||
import { getMaxBitrate } from '@shared/core-utils'
|
||||
import {
|
||||
ffprobePromise,
|
||||
getAudioStream,
|
||||
getMaxAudioBitrate,
|
||||
getVideoStream,
|
||||
getVideoStreamBitrate,
|
||||
getVideoStreamDimensionsInfo,
|
||||
getVideoStreamFPS
|
||||
} from '@shared/ffmpeg'
|
||||
|
||||
export async function canDoQuickTranscode (path: string, existingProbe?: FfprobeData): Promise<boolean> {
|
||||
if (CONFIG.TRANSCODING.PROFILE !== 'default') return false
|
||||
|
||||
const probe = existingProbe || await ffprobePromise(path)
|
||||
|
||||
return await canDoQuickVideoTranscode(path, probe) &&
|
||||
await canDoQuickAudioTranscode(path, probe)
|
||||
}
|
||||
|
||||
export async function canDoQuickAudioTranscode (path: string, probe?: FfprobeData): Promise<boolean> {
|
||||
const parsedAudio = await getAudioStream(path, probe)
|
||||
|
||||
if (!parsedAudio.audioStream) return true
|
||||
|
||||
if (parsedAudio.audioStream['codec_name'] !== 'aac') return false
|
||||
|
||||
const audioBitrate = parsedAudio.bitrate
|
||||
if (!audioBitrate) return false
|
||||
|
||||
const maxAudioBitrate = getMaxAudioBitrate('aac', audioBitrate)
|
||||
if (maxAudioBitrate !== -1 && audioBitrate > maxAudioBitrate) return false
|
||||
|
||||
const channelLayout = parsedAudio.audioStream['channel_layout']
|
||||
// Causes playback issues with Chrome
|
||||
if (!channelLayout || channelLayout === 'unknown' || channelLayout === 'quad') return false
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
export async function canDoQuickVideoTranscode (path: string, probe?: FfprobeData): Promise<boolean> {
|
||||
const videoStream = await getVideoStream(path, probe)
|
||||
const fps = await getVideoStreamFPS(path, probe)
|
||||
const bitRate = await getVideoStreamBitrate(path, probe)
|
||||
const resolutionData = await getVideoStreamDimensionsInfo(path, probe)
|
||||
|
||||
// If ffprobe did not manage to guess the bitrate
|
||||
if (!bitRate) return false
|
||||
|
||||
// check video params
|
||||
if (!videoStream) return false
|
||||
if (videoStream['codec_name'] !== 'h264') return false
|
||||
if (videoStream['pix_fmt'] !== 'yuv420p') return false
|
||||
if (fps < VIDEO_TRANSCODING_FPS.MIN || fps > VIDEO_TRANSCODING_FPS.MAX) return false
|
||||
if (bitRate > getMaxBitrate({ ...resolutionData, fps })) return false
|
||||
|
||||
return true
|
||||
}
|
52
server/lib/transcoding/transcoding-resolutions.ts
Normal file
52
server/lib/transcoding/transcoding-resolutions.ts
Normal file
@ -0,0 +1,52 @@
|
||||
import { CONFIG } from '@server/initializers/config'
|
||||
import { toEven } from '@shared/core-utils'
|
||||
import { VideoResolution } from '@shared/models'
|
||||
|
||||
export function computeResolutionsToTranscode (options: {
|
||||
input: number
|
||||
type: 'vod' | 'live'
|
||||
includeInput: boolean
|
||||
strictLower: boolean
|
||||
hasAudio: boolean
|
||||
}) {
|
||||
const { input, type, includeInput, strictLower, hasAudio } = options
|
||||
|
||||
const configResolutions = type === 'vod'
|
||||
? CONFIG.TRANSCODING.RESOLUTIONS
|
||||
: CONFIG.LIVE.TRANSCODING.RESOLUTIONS
|
||||
|
||||
const resolutionsEnabled = new Set<number>()
|
||||
|
||||
// Put in the order we want to proceed jobs
|
||||
const availableResolutions: VideoResolution[] = [
|
||||
VideoResolution.H_NOVIDEO,
|
||||
VideoResolution.H_480P,
|
||||
VideoResolution.H_360P,
|
||||
VideoResolution.H_720P,
|
||||
VideoResolution.H_240P,
|
||||
VideoResolution.H_144P,
|
||||
VideoResolution.H_1080P,
|
||||
VideoResolution.H_1440P,
|
||||
VideoResolution.H_4K
|
||||
]
|
||||
|
||||
for (const resolution of availableResolutions) {
|
||||
// Resolution not enabled
|
||||
if (configResolutions[resolution + 'p'] !== true) continue
|
||||
// Too big resolution for input file
|
||||
if (input < resolution) continue
|
||||
// We only want lower resolutions than input file
|
||||
if (strictLower && input === resolution) continue
|
||||
// Audio resolutio but no audio in the video
|
||||
if (resolution === VideoResolution.H_NOVIDEO && !hasAudio) continue
|
||||
|
||||
resolutionsEnabled.add(resolution)
|
||||
}
|
||||
|
||||
if (includeInput) {
|
||||
// Always use an even resolution to avoid issues with ffmpeg
|
||||
resolutionsEnabled.add(toEven(input))
|
||||
}
|
||||
|
||||
return Array.from(resolutionsEnabled)
|
||||
}
|
@ -1,465 +0,0 @@
|
||||
import { MutexInterface } from 'async-mutex'
|
||||
import { Job } from 'bullmq'
|
||||
import { copyFile, ensureDir, move, remove, stat } from 'fs-extra'
|
||||
import { basename, extname as extnameUtil, join } from 'path'
|
||||
import { toEven } from '@server/helpers/core-utils'
|
||||
import { retryTransactionWrapper } from '@server/helpers/database-utils'
|
||||
import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
|
||||
import { sequelizeTypescript } from '@server/initializers/database'
|
||||
import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
|
||||
import { pick } from '@shared/core-utils'
|
||||
import { VideoResolution, VideoStorage } from '../../../shared/models/videos'
|
||||
import {
|
||||
buildFileMetadata,
|
||||
canDoQuickTranscode,
|
||||
computeResolutionsToTranscode,
|
||||
ffprobePromise,
|
||||
getVideoStreamDuration,
|
||||
getVideoStreamFPS,
|
||||
transcodeVOD,
|
||||
TranscodeVODOptions,
|
||||
TranscodeVODOptionsType
|
||||
} from '../../helpers/ffmpeg'
|
||||
import { CONFIG } from '../../initializers/config'
|
||||
import { VideoFileModel } from '../../models/video/video-file'
|
||||
import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
|
||||
import { updatePlaylistAfterFileChange } from '../hls'
|
||||
import { generateHLSVideoFilename, generateWebTorrentVideoFilename, getHlsResolutionPlaylistFilename } from '../paths'
|
||||
import { VideoPathManager } from '../video-path-manager'
|
||||
import { VideoTranscodingProfilesManager } from './default-transcoding-profiles'
|
||||
|
||||
/**
|
||||
*
|
||||
* Functions that run transcoding functions, update the database, cleanup files, create torrent files...
|
||||
* Mainly called by the job queue
|
||||
*
|
||||
*/
|
||||
|
||||
// Optimize the original video file and replace it. The resolution is not changed.
|
||||
async function optimizeOriginalVideofile (options: {
|
||||
video: MVideoFullLight
|
||||
inputVideoFile: MVideoFile
|
||||
job: Job
|
||||
}) {
|
||||
const { video, inputVideoFile, job } = options
|
||||
|
||||
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
|
||||
const newExtname = '.mp4'
|
||||
|
||||
// Will be released by our transcodeVOD function once ffmpeg is ran
|
||||
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
|
||||
|
||||
try {
|
||||
await video.reload()
|
||||
|
||||
const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video)
|
||||
|
||||
const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async videoInputPath => {
|
||||
const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
|
||||
|
||||
const transcodeType: TranscodeVODOptionsType = await canDoQuickTranscode(videoInputPath)
|
||||
? 'quick-transcode'
|
||||
: 'video'
|
||||
|
||||
const resolution = buildOriginalFileResolution(inputVideoFile.resolution)
|
||||
|
||||
const transcodeOptions: TranscodeVODOptions = {
|
||||
type: transcodeType,
|
||||
|
||||
inputPath: videoInputPath,
|
||||
outputPath: videoTranscodedPath,
|
||||
|
||||
inputFileMutexReleaser,
|
||||
|
||||
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
|
||||
profile: CONFIG.TRANSCODING.PROFILE,
|
||||
|
||||
resolution,
|
||||
|
||||
job
|
||||
}
|
||||
|
||||
// Could be very long!
|
||||
await transcodeVOD(transcodeOptions)
|
||||
|
||||
// Important to do this before getVideoFilename() to take in account the new filename
|
||||
inputVideoFile.resolution = resolution
|
||||
inputVideoFile.extname = newExtname
|
||||
inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname)
|
||||
inputVideoFile.storage = VideoStorage.FILE_SYSTEM
|
||||
|
||||
const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile)
|
||||
await remove(videoInputPath)
|
||||
|
||||
return { transcodeType, videoFile }
|
||||
})
|
||||
|
||||
return result
|
||||
} finally {
|
||||
inputFileMutexReleaser()
|
||||
}
|
||||
}
|
||||
|
||||
// Transcode the original video file to a lower resolution compatible with WebTorrent
|
||||
async function transcodeNewWebTorrentResolution (options: {
|
||||
video: MVideoFullLight
|
||||
resolution: VideoResolution
|
||||
job: Job
|
||||
}) {
|
||||
const { video, resolution, job } = options
|
||||
|
||||
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
|
||||
const newExtname = '.mp4'
|
||||
|
||||
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
|
||||
|
||||
try {
|
||||
await video.reload()
|
||||
|
||||
const file = video.getMaxQualityFile().withVideoOrPlaylist(video)
|
||||
|
||||
const result = await VideoPathManager.Instance.makeAvailableVideoFile(file, async videoInputPath => {
|
||||
const newVideoFile = new VideoFileModel({
|
||||
resolution,
|
||||
extname: newExtname,
|
||||
filename: generateWebTorrentVideoFilename(resolution, newExtname),
|
||||
size: 0,
|
||||
videoId: video.id
|
||||
})
|
||||
|
||||
const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename)
|
||||
|
||||
const transcodeOptions = resolution === VideoResolution.H_NOVIDEO
|
||||
? {
|
||||
type: 'only-audio' as 'only-audio',
|
||||
|
||||
inputPath: videoInputPath,
|
||||
outputPath: videoTranscodedPath,
|
||||
|
||||
inputFileMutexReleaser,
|
||||
|
||||
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
|
||||
profile: CONFIG.TRANSCODING.PROFILE,
|
||||
|
||||
resolution,
|
||||
|
||||
job
|
||||
}
|
||||
: {
|
||||
type: 'video' as 'video',
|
||||
inputPath: videoInputPath,
|
||||
outputPath: videoTranscodedPath,
|
||||
|
||||
inputFileMutexReleaser,
|
||||
|
||||
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
|
||||
profile: CONFIG.TRANSCODING.PROFILE,
|
||||
|
||||
resolution,
|
||||
|
||||
job
|
||||
}
|
||||
|
||||
await transcodeVOD(transcodeOptions)
|
||||
|
||||
return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, newVideoFile)
|
||||
})
|
||||
|
||||
return result
|
||||
} finally {
|
||||
inputFileMutexReleaser()
|
||||
}
|
||||
}
|
||||
|
||||
// Merge an image with an audio file to create a video
|
||||
async function mergeAudioVideofile (options: {
|
||||
video: MVideoFullLight
|
||||
resolution: VideoResolution
|
||||
job: Job
|
||||
}) {
|
||||
const { video, resolution, job } = options
|
||||
|
||||
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
|
||||
const newExtname = '.mp4'
|
||||
|
||||
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
|
||||
|
||||
try {
|
||||
await video.reload()
|
||||
|
||||
const inputVideoFile = video.getMinQualityFile()
|
||||
|
||||
const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video)
|
||||
|
||||
const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async audioInputPath => {
|
||||
const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
|
||||
|
||||
// If the user updates the video preview during transcoding
|
||||
const previewPath = video.getPreview().getPath()
|
||||
const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath))
|
||||
await copyFile(previewPath, tmpPreviewPath)
|
||||
|
||||
const transcodeOptions = {
|
||||
type: 'merge-audio' as 'merge-audio',
|
||||
|
||||
inputPath: tmpPreviewPath,
|
||||
outputPath: videoTranscodedPath,
|
||||
|
||||
inputFileMutexReleaser,
|
||||
|
||||
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
|
||||
profile: CONFIG.TRANSCODING.PROFILE,
|
||||
|
||||
audioPath: audioInputPath,
|
||||
resolution,
|
||||
|
||||
job
|
||||
}
|
||||
|
||||
try {
|
||||
await transcodeVOD(transcodeOptions)
|
||||
|
||||
await remove(audioInputPath)
|
||||
await remove(tmpPreviewPath)
|
||||
} catch (err) {
|
||||
await remove(tmpPreviewPath)
|
||||
throw err
|
||||
}
|
||||
|
||||
// Important to do this before getVideoFilename() to take in account the new file extension
|
||||
inputVideoFile.extname = newExtname
|
||||
inputVideoFile.resolution = resolution
|
||||
inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname)
|
||||
|
||||
// ffmpeg generated a new video file, so update the video duration
|
||||
// See https://trac.ffmpeg.org/ticket/5456
|
||||
video.duration = await getVideoStreamDuration(videoTranscodedPath)
|
||||
await video.save()
|
||||
|
||||
return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile)
|
||||
})
|
||||
|
||||
return result
|
||||
} finally {
|
||||
inputFileMutexReleaser()
|
||||
}
|
||||
}
|
||||
|
||||
// Concat TS segments from a live video to a fragmented mp4 HLS playlist
|
||||
async function generateHlsPlaylistResolutionFromTS (options: {
|
||||
video: MVideo
|
||||
concatenatedTsFilePath: string
|
||||
resolution: VideoResolution
|
||||
isAAC: boolean
|
||||
inputFileMutexReleaser: MutexInterface.Releaser
|
||||
}) {
|
||||
return generateHlsPlaylistCommon({
|
||||
type: 'hls-from-ts' as 'hls-from-ts',
|
||||
inputPath: options.concatenatedTsFilePath,
|
||||
|
||||
...pick(options, [ 'video', 'resolution', 'inputFileMutexReleaser', 'isAAC' ])
|
||||
})
|
||||
}
|
||||
|
||||
// Generate an HLS playlist from an input file, and update the master playlist
|
||||
function generateHlsPlaylistResolution (options: {
|
||||
video: MVideo
|
||||
videoInputPath: string
|
||||
resolution: VideoResolution
|
||||
copyCodecs: boolean
|
||||
inputFileMutexReleaser: MutexInterface.Releaser
|
||||
job?: Job
|
||||
}) {
|
||||
return generateHlsPlaylistCommon({
|
||||
type: 'hls' as 'hls',
|
||||
inputPath: options.videoInputPath,
|
||||
|
||||
...pick(options, [ 'video', 'resolution', 'copyCodecs', 'inputFileMutexReleaser', 'job' ])
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
generateHlsPlaylistResolution,
|
||||
generateHlsPlaylistResolutionFromTS,
|
||||
optimizeOriginalVideofile,
|
||||
transcodeNewWebTorrentResolution,
|
||||
mergeAudioVideofile
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function onWebTorrentVideoFileTranscoding (
|
||||
video: MVideoFullLight,
|
||||
videoFile: MVideoFile,
|
||||
transcodingPath: string,
|
||||
newVideoFile: MVideoFile
|
||||
) {
|
||||
const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
|
||||
|
||||
try {
|
||||
await video.reload()
|
||||
|
||||
const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile)
|
||||
|
||||
const stats = await stat(transcodingPath)
|
||||
|
||||
const probe = await ffprobePromise(transcodingPath)
|
||||
const fps = await getVideoStreamFPS(transcodingPath, probe)
|
||||
const metadata = await buildFileMetadata(transcodingPath, probe)
|
||||
|
||||
await move(transcodingPath, outputPath, { overwrite: true })
|
||||
|
||||
videoFile.size = stats.size
|
||||
videoFile.fps = fps
|
||||
videoFile.metadata = metadata
|
||||
|
||||
await createTorrentAndSetInfoHash(video, videoFile)
|
||||
|
||||
const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution })
|
||||
if (oldFile) await video.removeWebTorrentFile(oldFile)
|
||||
|
||||
await VideoFileModel.customUpsert(videoFile, 'video', undefined)
|
||||
video.VideoFiles = await video.$get('VideoFiles')
|
||||
|
||||
return { video, videoFile }
|
||||
} finally {
|
||||
mutexReleaser()
|
||||
}
|
||||
}
|
||||
|
||||
async function generateHlsPlaylistCommon (options: {
|
||||
type: 'hls' | 'hls-from-ts'
|
||||
video: MVideo
|
||||
inputPath: string
|
||||
resolution: VideoResolution
|
||||
|
||||
inputFileMutexReleaser: MutexInterface.Releaser
|
||||
|
||||
copyCodecs?: boolean
|
||||
isAAC?: boolean
|
||||
|
||||
job?: Job
|
||||
}) {
|
||||
const { type, video, inputPath, resolution, copyCodecs, isAAC, job, inputFileMutexReleaser } = options
|
||||
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
|
||||
|
||||
const videoTranscodedBasePath = join(transcodeDirectory, type)
|
||||
await ensureDir(videoTranscodedBasePath)
|
||||
|
||||
const videoFilename = generateHLSVideoFilename(resolution)
|
||||
const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFilename)
|
||||
const resolutionPlaylistFileTranscodePath = join(videoTranscodedBasePath, resolutionPlaylistFilename)
|
||||
|
||||
const transcodeOptions = {
|
||||
type,
|
||||
|
||||
inputPath,
|
||||
outputPath: resolutionPlaylistFileTranscodePath,
|
||||
|
||||
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
|
||||
profile: CONFIG.TRANSCODING.PROFILE,
|
||||
|
||||
resolution,
|
||||
copyCodecs,
|
||||
|
||||
isAAC,
|
||||
|
||||
inputFileMutexReleaser,
|
||||
|
||||
hlsPlaylist: {
|
||||
videoFilename
|
||||
},
|
||||
|
||||
job
|
||||
}
|
||||
|
||||
await transcodeVOD(transcodeOptions)
|
||||
|
||||
// Create or update the playlist
|
||||
const playlist = await retryTransactionWrapper(() => {
|
||||
return sequelizeTypescript.transaction(async transaction => {
|
||||
return VideoStreamingPlaylistModel.loadOrGenerate(video, transaction)
|
||||
})
|
||||
})
|
||||
|
||||
const newVideoFile = new VideoFileModel({
|
||||
resolution,
|
||||
extname: extnameUtil(videoFilename),
|
||||
size: 0,
|
||||
filename: videoFilename,
|
||||
fps: -1,
|
||||
videoStreamingPlaylistId: playlist.id
|
||||
})
|
||||
|
||||
const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
|
||||
|
||||
try {
|
||||
// VOD transcoding is a long task, refresh video attributes
|
||||
await video.reload()
|
||||
|
||||
const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile)
|
||||
await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video))
|
||||
|
||||
// Move playlist file
|
||||
const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename)
|
||||
await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true })
|
||||
// Move video file
|
||||
await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true })
|
||||
|
||||
// Update video duration if it was not set (in case of a live for example)
|
||||
if (!video.duration) {
|
||||
video.duration = await getVideoStreamDuration(videoFilePath)
|
||||
await video.save()
|
||||
}
|
||||
|
||||
const stats = await stat(videoFilePath)
|
||||
|
||||
newVideoFile.size = stats.size
|
||||
newVideoFile.fps = await getVideoStreamFPS(videoFilePath)
|
||||
newVideoFile.metadata = await buildFileMetadata(videoFilePath)
|
||||
|
||||
await createTorrentAndSetInfoHash(playlist, newVideoFile)
|
||||
|
||||
const oldFile = await VideoFileModel.loadHLSFile({
|
||||
playlistId: playlist.id,
|
||||
fps: newVideoFile.fps,
|
||||
resolution: newVideoFile.resolution
|
||||
})
|
||||
|
||||
if (oldFile) {
|
||||
await video.removeStreamingPlaylistVideoFile(playlist, oldFile)
|
||||
await oldFile.destroy()
|
||||
}
|
||||
|
||||
const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)
|
||||
|
||||
await updatePlaylistAfterFileChange(video, playlist)
|
||||
|
||||
return { resolutionPlaylistPath, videoFile: savedVideoFile }
|
||||
} finally {
|
||||
mutexReleaser()
|
||||
}
|
||||
}
|
||||
|
||||
function buildOriginalFileResolution (inputResolution: number) {
|
||||
if (CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION === true) {
|
||||
return toEven(inputResolution)
|
||||
}
|
||||
|
||||
const resolutions = computeResolutionsToTranscode({
|
||||
input: inputResolution,
|
||||
type: 'vod',
|
||||
includeInput: false,
|
||||
strictLower: false,
|
||||
// We don't really care about the audio resolution in this context
|
||||
hasAudio: true
|
||||
})
|
||||
|
||||
if (resolutions.length === 0) {
|
||||
return toEven(inputResolution)
|
||||
}
|
||||
|
||||
return Math.max(...resolutions)
|
||||
}
|
273
server/lib/transcoding/web-transcoding.ts
Normal file
273
server/lib/transcoding/web-transcoding.ts
Normal file
@ -0,0 +1,273 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { copyFile, move, remove, stat } from 'fs-extra'
|
||||
import { basename, join } from 'path'
|
||||
import { computeOutputFPS } from '@server/helpers/ffmpeg'
|
||||
import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
|
||||
import { MVideoFile, MVideoFullLight } from '@server/types/models'
|
||||
import { toEven } from '@shared/core-utils'
|
||||
import { ffprobePromise, getVideoStreamDuration, getVideoStreamFPS, TranscodeVODOptionsType } from '@shared/ffmpeg'
|
||||
import { VideoResolution, VideoStorage } from '@shared/models'
|
||||
import { CONFIG } from '../../initializers/config'
|
||||
import { VideoFileModel } from '../../models/video/video-file'
|
||||
import { generateWebTorrentVideoFilename } from '../paths'
|
||||
import { buildFileMetadata } from '../video-file'
|
||||
import { VideoPathManager } from '../video-path-manager'
|
||||
import { buildFFmpegVOD } from './shared'
|
||||
import { computeResolutionsToTranscode } from './transcoding-resolutions'
|
||||
|
||||
// Optimize the original video file and replace it. The resolution is not changed.
|
||||
export async function optimizeOriginalVideofile (options: {
|
||||
video: MVideoFullLight
|
||||
inputVideoFile: MVideoFile
|
||||
quickTranscode: boolean
|
||||
job: Job
|
||||
}) {
|
||||
const { video, inputVideoFile, quickTranscode, job } = options
|
||||
|
||||
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
|
||||
const newExtname = '.mp4'
|
||||
|
||||
// Will be released by our transcodeVOD function once ffmpeg is ran
|
||||
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
|
||||
|
||||
try {
|
||||
await video.reload()
|
||||
|
||||
const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video)
|
||||
|
||||
const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async videoInputPath => {
|
||||
const videoOutputPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
|
||||
|
||||
const transcodeType: TranscodeVODOptionsType = quickTranscode
|
||||
? 'quick-transcode'
|
||||
: 'video'
|
||||
|
||||
const resolution = buildOriginalFileResolution(inputVideoFile.resolution)
|
||||
const fps = computeOutputFPS({ inputFPS: inputVideoFile.fps, resolution })
|
||||
|
||||
// Could be very long!
|
||||
await buildFFmpegVOD(job).transcode({
|
||||
type: transcodeType,
|
||||
|
||||
inputPath: videoInputPath,
|
||||
outputPath: videoOutputPath,
|
||||
|
||||
inputFileMutexReleaser,
|
||||
|
||||
resolution,
|
||||
fps
|
||||
})
|
||||
|
||||
// Important to do this before getVideoFilename() to take in account the new filename
|
||||
inputVideoFile.resolution = resolution
|
||||
inputVideoFile.extname = newExtname
|
||||
inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname)
|
||||
inputVideoFile.storage = VideoStorage.FILE_SYSTEM
|
||||
|
||||
const { videoFile } = await onWebTorrentVideoFileTranscoding({
|
||||
video,
|
||||
videoFile: inputVideoFile,
|
||||
videoOutputPath
|
||||
})
|
||||
|
||||
await remove(videoInputPath)
|
||||
|
||||
return { transcodeType, videoFile }
|
||||
})
|
||||
|
||||
return result
|
||||
} finally {
|
||||
inputFileMutexReleaser()
|
||||
}
|
||||
}
|
||||
|
||||
// Transcode the original video file to a lower resolution compatible with WebTorrent
|
||||
export async function transcodeNewWebTorrentResolution (options: {
|
||||
video: MVideoFullLight
|
||||
resolution: VideoResolution
|
||||
fps: number
|
||||
job: Job
|
||||
}) {
|
||||
const { video, resolution, fps, job } = options
|
||||
|
||||
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
|
||||
const newExtname = '.mp4'
|
||||
|
||||
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
|
||||
|
||||
try {
|
||||
await video.reload()
|
||||
|
||||
const file = video.getMaxQualityFile().withVideoOrPlaylist(video)
|
||||
|
||||
const result = await VideoPathManager.Instance.makeAvailableVideoFile(file, async videoInputPath => {
|
||||
const newVideoFile = new VideoFileModel({
|
||||
resolution,
|
||||
extname: newExtname,
|
||||
filename: generateWebTorrentVideoFilename(resolution, newExtname),
|
||||
size: 0,
|
||||
videoId: video.id
|
||||
})
|
||||
|
||||
const videoOutputPath = join(transcodeDirectory, newVideoFile.filename)
|
||||
|
||||
const transcodeOptions = {
|
||||
type: 'video' as 'video',
|
||||
|
||||
inputPath: videoInputPath,
|
||||
outputPath: videoOutputPath,
|
||||
|
||||
inputFileMutexReleaser,
|
||||
|
||||
resolution,
|
||||
fps
|
||||
}
|
||||
|
||||
await buildFFmpegVOD(job).transcode(transcodeOptions)
|
||||
|
||||
return onWebTorrentVideoFileTranscoding({ video, videoFile: newVideoFile, videoOutputPath })
|
||||
})
|
||||
|
||||
return result
|
||||
} finally {
|
||||
inputFileMutexReleaser()
|
||||
}
|
||||
}
|
||||
|
||||
// Merge an image with an audio file to create a video
|
||||
export async function mergeAudioVideofile (options: {
|
||||
video: MVideoFullLight
|
||||
resolution: VideoResolution
|
||||
fps: number
|
||||
job: Job
|
||||
}) {
|
||||
const { video, resolution, fps, job } = options
|
||||
|
||||
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
|
||||
const newExtname = '.mp4'
|
||||
|
||||
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
|
||||
|
||||
try {
|
||||
await video.reload()
|
||||
|
||||
const inputVideoFile = video.getMinQualityFile()
|
||||
|
||||
const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video)
|
||||
|
||||
const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async audioInputPath => {
|
||||
const videoOutputPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
|
||||
|
||||
// If the user updates the video preview during transcoding
|
||||
const previewPath = video.getPreview().getPath()
|
||||
const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath))
|
||||
await copyFile(previewPath, tmpPreviewPath)
|
||||
|
||||
const transcodeOptions = {
|
||||
type: 'merge-audio' as 'merge-audio',
|
||||
|
||||
inputPath: tmpPreviewPath,
|
||||
outputPath: videoOutputPath,
|
||||
|
||||
inputFileMutexReleaser,
|
||||
|
||||
audioPath: audioInputPath,
|
||||
resolution,
|
||||
fps
|
||||
}
|
||||
|
||||
try {
|
||||
await buildFFmpegVOD(job).transcode(transcodeOptions)
|
||||
|
||||
await remove(audioInputPath)
|
||||
await remove(tmpPreviewPath)
|
||||
} catch (err) {
|
||||
await remove(tmpPreviewPath)
|
||||
throw err
|
||||
}
|
||||
|
||||
// Important to do this before getVideoFilename() to take in account the new file extension
|
||||
inputVideoFile.extname = newExtname
|
||||
inputVideoFile.resolution = resolution
|
||||
inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname)
|
||||
|
||||
// ffmpeg generated a new video file, so update the video duration
|
||||
// See https://trac.ffmpeg.org/ticket/5456
|
||||
video.duration = await getVideoStreamDuration(videoOutputPath)
|
||||
await video.save()
|
||||
|
||||
return onWebTorrentVideoFileTranscoding({
|
||||
video,
|
||||
videoFile: inputVideoFile,
|
||||
videoOutputPath
|
||||
})
|
||||
})
|
||||
|
||||
return result
|
||||
} finally {
|
||||
inputFileMutexReleaser()
|
||||
}
|
||||
}
|
||||
|
||||
export async function onWebTorrentVideoFileTranscoding (options: {
|
||||
video: MVideoFullLight
|
||||
videoFile: MVideoFile
|
||||
videoOutputPath: string
|
||||
}) {
|
||||
const { video, videoFile, videoOutputPath } = options
|
||||
|
||||
const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
|
||||
|
||||
try {
|
||||
await video.reload()
|
||||
|
||||
const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile)
|
||||
|
||||
const stats = await stat(videoOutputPath)
|
||||
|
||||
const probe = await ffprobePromise(videoOutputPath)
|
||||
const fps = await getVideoStreamFPS(videoOutputPath, probe)
|
||||
const metadata = await buildFileMetadata(videoOutputPath, probe)
|
||||
|
||||
await move(videoOutputPath, outputPath, { overwrite: true })
|
||||
|
||||
videoFile.size = stats.size
|
||||
videoFile.fps = fps
|
||||
videoFile.metadata = metadata
|
||||
|
||||
await createTorrentAndSetInfoHash(video, videoFile)
|
||||
|
||||
const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution })
|
||||
if (oldFile) await video.removeWebTorrentFile(oldFile)
|
||||
|
||||
await VideoFileModel.customUpsert(videoFile, 'video', undefined)
|
||||
video.VideoFiles = await video.$get('VideoFiles')
|
||||
|
||||
return { video, videoFile }
|
||||
} finally {
|
||||
mutexReleaser()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function buildOriginalFileResolution (inputResolution: number) {
|
||||
if (CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION === true) {
|
||||
return toEven(inputResolution)
|
||||
}
|
||||
|
||||
const resolutions = computeResolutionsToTranscode({
|
||||
input: inputResolution,
|
||||
type: 'vod',
|
||||
includeInput: false,
|
||||
strictLower: false,
|
||||
// We don't really care about the audio resolution in this context
|
||||
hasAudio: true
|
||||
})
|
||||
|
||||
if (resolutions.length === 0) {
|
||||
return toEven(inputResolution)
|
||||
}
|
||||
|
||||
return Math.max(...resolutions)
|
||||
}
|
@ -3,6 +3,7 @@ import { buildLogger } from '@server/helpers/logger'
|
||||
import { getResumableUploadPath } from '@server/helpers/upload'
|
||||
import { CONFIG } from '@server/initializers/config'
|
||||
import { LogLevel, Uploadx } from '@uploadx/core'
|
||||
import { extname } from 'path'
|
||||
|
||||
const logger = buildLogger('uploadx')
|
||||
|
||||
@ -26,7 +27,9 @@ const uploadx = new Uploadx({
|
||||
if (!res.locals.oauth) return undefined
|
||||
|
||||
return res.locals.oauth.token.user.id + ''
|
||||
}
|
||||
},
|
||||
|
||||
filename: file => `${file.userId}-${file.id}${extname(file.metadata.filename)}`
|
||||
})
|
||||
|
||||
export {
|
||||
|
@ -81,7 +81,7 @@ async function blacklistVideo (videoInstance: MVideoAccountLight, options: Video
|
||||
}
|
||||
|
||||
if (videoInstance.isLive) {
|
||||
LiveManager.Instance.stopSessionOf(videoInstance.id, LiveVideoError.BLACKLISTED)
|
||||
LiveManager.Instance.stopSessionOf(videoInstance.uuid, LiveVideoError.BLACKLISTED)
|
||||
}
|
||||
|
||||
Notifier.Instance.notifyOnVideoBlacklist(blacklist)
|
||||
|
@ -1,6 +1,44 @@
|
||||
import { FfprobeData } from 'fluent-ffmpeg'
|
||||
import { logger } from '@server/helpers/logger'
|
||||
import { VideoFileModel } from '@server/models/video/video-file'
|
||||
import { MVideoWithAllFiles } from '@server/types/models'
|
||||
import { getLowercaseExtension } from '@shared/core-utils'
|
||||
import { getFileSize } from '@shared/extra-utils'
|
||||
import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, isAudioFile } from '@shared/ffmpeg'
|
||||
import { VideoFileMetadata, VideoResolution } from '@shared/models'
|
||||
import { lTags } from './object-storage/shared'
|
||||
import { generateHLSVideoFilename, generateWebTorrentVideoFilename } from './paths'
|
||||
|
||||
async function buildNewFile (options: {
|
||||
path: string
|
||||
mode: 'web-video' | 'hls'
|
||||
}) {
|
||||
const { path, mode } = options
|
||||
|
||||
const probe = await ffprobePromise(path)
|
||||
const size = await getFileSize(path)
|
||||
|
||||
const videoFile = new VideoFileModel({
|
||||
extname: getLowercaseExtension(path),
|
||||
size,
|
||||
metadata: await buildFileMetadata(path, probe)
|
||||
})
|
||||
|
||||
if (await isAudioFile(path, probe)) {
|
||||
videoFile.resolution = VideoResolution.H_NOVIDEO
|
||||
} else {
|
||||
videoFile.fps = await getVideoStreamFPS(path, probe)
|
||||
videoFile.resolution = (await getVideoStreamDimensionsInfo(path, probe)).resolution
|
||||
}
|
||||
|
||||
videoFile.filename = mode === 'web-video'
|
||||
? generateWebTorrentVideoFilename(videoFile.resolution, videoFile.extname)
|
||||
: generateHLSVideoFilename(videoFile.resolution)
|
||||
|
||||
return videoFile
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function removeHLSPlaylist (video: MVideoWithAllFiles) {
|
||||
const hls = video.getHLSPlaylist()
|
||||
@ -61,9 +99,23 @@ async function removeWebTorrentFile (video: MVideoWithAllFiles, fileToDeleteId:
|
||||
return video
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function buildFileMetadata (path: string, existingProbe?: FfprobeData) {
|
||||
const metadata = existingProbe || await ffprobePromise(path)
|
||||
|
||||
return new VideoFileMetadata(metadata)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
buildNewFile,
|
||||
|
||||
removeHLSPlaylist,
|
||||
removeHLSFile,
|
||||
removeAllWebTorrentFiles,
|
||||
removeWebTorrentFile
|
||||
removeWebTorrentFile,
|
||||
|
||||
buildFileMetadata
|
||||
}
|
||||
|
@ -1,5 +1,5 @@
|
||||
import { MVideoFullLight } from '@server/types/models'
|
||||
import { getVideoStreamDuration } from '@shared/extra-utils'
|
||||
import { getVideoStreamDuration } from '@shared/ffmpeg'
|
||||
import { VideoStudioTask } from '@shared/models'
|
||||
|
||||
function buildTaskFileFieldname (indice: number, fieldName = 'file') {
|
||||
|
@ -2,14 +2,14 @@ import { UploadFiles } from 'express'
|
||||
import memoizee from 'memoizee'
|
||||
import { Transaction } from 'sequelize/types'
|
||||
import { CONFIG } from '@server/initializers/config'
|
||||
import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY, MEMOIZE_LENGTH, MEMOIZE_TTL } from '@server/initializers/constants'
|
||||
import { MEMOIZE_LENGTH, MEMOIZE_TTL } from '@server/initializers/constants'
|
||||
import { TagModel } from '@server/models/video/tag'
|
||||
import { VideoModel } from '@server/models/video/video'
|
||||
import { VideoJobInfoModel } from '@server/models/video/video-job-info'
|
||||
import { FilteredModelAttributes } from '@server/types'
|
||||
import { MThumbnail, MUserId, MVideoFile, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
|
||||
import { ManageVideoTorrentPayload, ThumbnailType, VideoCreate, VideoPrivacy, VideoState, VideoTranscodingPayload } from '@shared/models'
|
||||
import { CreateJobArgument, CreateJobOptions, JobQueue } from './job-queue/job-queue'
|
||||
import { MThumbnail, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
|
||||
import { ManageVideoTorrentPayload, ThumbnailType, VideoCreate, VideoPrivacy, VideoState } from '@shared/models'
|
||||
import { CreateJobArgument, JobQueue } from './job-queue/job-queue'
|
||||
import { updateVideoMiniatureFromExisting } from './thumbnail'
|
||||
import { moveFilesIfPrivacyChanged } from './video-privacy'
|
||||
|
||||
@ -87,58 +87,6 @@ async function setVideoTags (options: {
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function buildOptimizeOrMergeAudioJob (options: {
|
||||
video: MVideoUUID
|
||||
videoFile: MVideoFile
|
||||
user: MUserId
|
||||
isNewVideo?: boolean // Default true
|
||||
}) {
|
||||
const { video, videoFile, user, isNewVideo } = options
|
||||
|
||||
let payload: VideoTranscodingPayload
|
||||
|
||||
if (videoFile.isAudio()) {
|
||||
payload = {
|
||||
type: 'merge-audio-to-webtorrent',
|
||||
resolution: DEFAULT_AUDIO_RESOLUTION,
|
||||
videoUUID: video.uuid,
|
||||
createHLSIfNeeded: true,
|
||||
isNewVideo
|
||||
}
|
||||
} else {
|
||||
payload = {
|
||||
type: 'optimize-to-webtorrent',
|
||||
videoUUID: video.uuid,
|
||||
isNewVideo
|
||||
}
|
||||
}
|
||||
|
||||
await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode')
|
||||
|
||||
return {
|
||||
type: 'video-transcoding' as 'video-transcoding',
|
||||
priority: await getTranscodingJobPriority(user),
|
||||
payload
|
||||
}
|
||||
}
|
||||
|
||||
async function buildTranscodingJob (payload: VideoTranscodingPayload, options: CreateJobOptions = {}) {
|
||||
await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode')
|
||||
|
||||
return { type: 'video-transcoding' as 'video-transcoding', payload, ...options }
|
||||
}
|
||||
|
||||
async function getTranscodingJobPriority (user: MUserId) {
|
||||
const now = new Date()
|
||||
const lastWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7)
|
||||
|
||||
const videoUploadedByUser = await VideoModel.countVideosUploadedByUserSince(user.id, lastWeek)
|
||||
|
||||
return JOB_PRIORITY.TRANSCODING + videoUploadedByUser
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function buildMoveToObjectStorageJob (options: {
|
||||
video: MVideoUUID
|
||||
previousVideoState: VideoState
|
||||
@ -235,10 +183,7 @@ export {
|
||||
buildLocalVideoFromReq,
|
||||
buildVideoThumbnailsFromReq,
|
||||
setVideoTags,
|
||||
buildOptimizeOrMergeAudioJob,
|
||||
buildTranscodingJob,
|
||||
buildMoveToObjectStorageJob,
|
||||
getTranscodingJobPriority,
|
||||
addVideoJobsAfterUpdate,
|
||||
getCachedVideoDuration
|
||||
}
|
||||
|
@ -1,6 +1,7 @@
|
||||
import express from 'express'
|
||||
import { Socket } from 'socket.io'
|
||||
import { getAccessToken } from '@server/lib/auth/oauth-model'
|
||||
import { RunnerModel } from '@server/models/runner/runner'
|
||||
import { HttpStatusCode } from '../../shared/models/http/http-error-codes'
|
||||
import { logger } from '../helpers/logger'
|
||||
import { handleOAuthAuthenticate } from '../lib/auth/oauth'
|
||||
@ -27,7 +28,7 @@ function authenticate (req: express.Request, res: express.Response, next: expres
|
||||
function authenticateSocket (socket: Socket, next: (err?: any) => void) {
|
||||
const accessToken = socket.handshake.query['accessToken']
|
||||
|
||||
logger.debug('Checking socket access token %s.', accessToken)
|
||||
logger.debug('Checking access token in runner.')
|
||||
|
||||
if (!accessToken) return next(new Error('No access token provided'))
|
||||
if (typeof accessToken !== 'string') return next(new Error('Access token is invalid'))
|
||||
@ -73,9 +74,31 @@ function optionalAuthenticate (req: express.Request, res: express.Response, next
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function authenticateRunnerSocket (socket: Socket, next: (err?: any) => void) {
|
||||
const runnerToken = socket.handshake.auth['runnerToken']
|
||||
|
||||
logger.debug('Checking runner token in socket.')
|
||||
|
||||
if (!runnerToken) return next(new Error('No runner token provided'))
|
||||
if (typeof runnerToken !== 'string') return next(new Error('Runner token is invalid'))
|
||||
|
||||
RunnerModel.loadByToken(runnerToken)
|
||||
.then(runner => {
|
||||
if (!runner) return next(new Error('Invalid runner token.'))
|
||||
|
||||
socket.handshake.auth.runner = runner
|
||||
|
||||
return next()
|
||||
})
|
||||
.catch(err => logger.error('Cannot get runner token.', { err }))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
authenticate,
|
||||
authenticateSocket,
|
||||
authenticatePromise,
|
||||
optionalAuthenticate
|
||||
optionalAuthenticate,
|
||||
authenticateRunnerSocket
|
||||
}
|
||||
|
@ -5,7 +5,7 @@ function openapiOperationDoc (options: {
|
||||
operationId?: string
|
||||
}) {
|
||||
return (req: express.Request, res: express.Response, next: express.NextFunction) => {
|
||||
res.locals.docUrl = options.url || 'https://docs.joinpeertube.org/api/rest-reference.html#operation/' + options.operationId
|
||||
res.locals.docUrl = options.url || 'https://docs.joinpeertube.org/api-rest-reference.html#operation/' + options.operationId
|
||||
|
||||
if (next) return next()
|
||||
}
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user