mirror of
https://github.com/Chocobozzz/PeerTube.git
synced 2024-09-11 21:47:09 +03:00
Support CLI move of original video file
This commit is contained in:
parent
54c140c800
commit
96b9748585
@ -166,9 +166,10 @@ export function makePutBodyRequest (options: {
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export async function getRedirectionUrl (url: string) {
|
||||
export async function getRedirectionUrl (url: string, token?: string) {
|
||||
const res = await makeRawRequest({
|
||||
url,
|
||||
token,
|
||||
redirects: 0,
|
||||
expectedStatus: HttpStatusCode.FOUND_302
|
||||
})
|
||||
|
@ -1,23 +1,26 @@
|
||||
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
|
||||
|
||||
import { join } from 'path'
|
||||
import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils'
|
||||
import { getAllFiles } from '@peertube/peertube-core-utils'
|
||||
import { HttpStatusCode, VideoDetails } from '@peertube/peertube-models'
|
||||
import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils'
|
||||
import {
|
||||
ObjectStorageCommand,
|
||||
PeerTubeServer,
|
||||
cleanupTests,
|
||||
createMultipleServers,
|
||||
doubleFollow,
|
||||
getRedirectionUrl,
|
||||
makeRawRequest,
|
||||
ObjectStorageCommand,
|
||||
PeerTubeServer,
|
||||
setAccessTokensToServers,
|
||||
waitJobs
|
||||
} from '@peertube/peertube-server-commands'
|
||||
import { expectStartWith } from '../shared/checks.js'
|
||||
import { checkDirectoryIsEmpty } from '@tests/shared/directories.js'
|
||||
import { getAllFiles } from '@peertube/peertube-core-utils'
|
||||
import { join } from 'path'
|
||||
import { expectStartWith } from '../shared/checks.js'
|
||||
|
||||
async function checkFiles (origin: PeerTubeServer, video: VideoDetails, objectStorage?: ObjectStorageCommand) {
|
||||
|
||||
// Web videos
|
||||
for (const file of video.files) {
|
||||
const start = objectStorage
|
||||
? objectStorage.getMockWebVideosBaseUrl()
|
||||
@ -28,18 +31,36 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, objectSt
|
||||
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
|
||||
}
|
||||
|
||||
const start = objectStorage
|
||||
? objectStorage.getMockPlaylistBaseUrl()
|
||||
: origin.url
|
||||
// Playlists
|
||||
{
|
||||
const start = objectStorage
|
||||
? objectStorage.getMockPlaylistBaseUrl()
|
||||
: origin.url
|
||||
|
||||
const hls = video.streamingPlaylists[0]
|
||||
expectStartWith(hls.playlistUrl, start)
|
||||
expectStartWith(hls.segmentsSha256Url, start)
|
||||
const hls = video.streamingPlaylists[0]
|
||||
expectStartWith(hls.playlistUrl, start)
|
||||
expectStartWith(hls.segmentsSha256Url, start)
|
||||
|
||||
for (const file of hls.files) {
|
||||
expectStartWith(file.fileUrl, start)
|
||||
for (const file of hls.files) {
|
||||
expectStartWith(file.fileUrl, start)
|
||||
|
||||
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
|
||||
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
|
||||
}
|
||||
}
|
||||
|
||||
// Original file
|
||||
{
|
||||
const source = await origin.videos.getSource({ id: video.uuid })
|
||||
|
||||
if (objectStorage) {
|
||||
await makeRawRequest({ url: source.fileDownloadUrl, token: origin.accessToken, expectedStatus: HttpStatusCode.FOUND_302 })
|
||||
|
||||
const redirected = await getRedirectionUrl(source.fileDownloadUrl, origin.accessToken)
|
||||
expectStartWith(redirected, objectStorage.getMockOriginalFileBaseUrl())
|
||||
} else {
|
||||
await makeRawRequest({ url: source.fileDownloadUrl, token: origin.accessToken, expectedStatus: HttpStatusCode.OK_200 })
|
||||
expectStartWith(source.fileDownloadUrl, origin.url)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -61,10 +82,10 @@ describe('Test create move video storage job', function () {
|
||||
|
||||
await objectStorage.prepareDefaultMockBuckets()
|
||||
|
||||
await servers[0].config.enableTranscoding()
|
||||
await servers[0].config.enableMinimumTranscoding({ keepOriginal: true })
|
||||
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video' + i } })
|
||||
const { uuid } = await servers[0].videos.quickUpload({ name: 'video' + i })
|
||||
uuids.push(uuid)
|
||||
}
|
||||
|
||||
|
@ -1,20 +1,23 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { join } from 'path'
|
||||
import { MoveStoragePayload, VideoStateType, FileStorage } from '@peertube/peertube-models'
|
||||
import { FileStorage, MoveStoragePayload, VideoStateType } from '@peertube/peertube-models'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { updateTorrentMetadata } from '@server/helpers/webtorrent.js'
|
||||
import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
|
||||
import {
|
||||
makeHLSFileAvailable,
|
||||
makeOriginalFileAvailable,
|
||||
makeWebVideoFileAvailable,
|
||||
removeHLSFileObjectStorageByFilename,
|
||||
removeHLSObjectStorage,
|
||||
removeOriginalFileObjectStorage,
|
||||
removeWebVideoObjectStorage
|
||||
} from '@server/lib/object-storage/index.js'
|
||||
import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { moveToFailedMoveToFileSystemState, moveToNextState } from '@server/lib/video-state.js'
|
||||
import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
|
||||
import { MVideoSource } from '@server/types/models/video/video-source.js'
|
||||
import { Job } from 'bullmq'
|
||||
import { join } from 'path'
|
||||
import { moveToJob, onMoveToStorageFailure } from './shared/move-video.js'
|
||||
|
||||
const lTagsBase = loggerTagsFactory('move-file-system')
|
||||
@ -30,6 +33,8 @@ export async function processMoveToFileSystem (job: Job) {
|
||||
|
||||
moveWebVideoFiles,
|
||||
moveHLSFiles,
|
||||
moveVideoSourceFile,
|
||||
|
||||
doAfterLastMove: video => doAfterLastMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }),
|
||||
moveToFailedState: moveToFailedMoveToFileSystemState
|
||||
})
|
||||
@ -50,12 +55,31 @@ export async function onMoveToFileSystemFailure (job: Job, err: any) {
|
||||
// Private
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function moveVideoSourceFile (source: MVideoSource) {
|
||||
if (source.storage === FileStorage.FILE_SYSTEM) return
|
||||
|
||||
await makeOriginalFileAvailable(
|
||||
source.keptOriginalFilename,
|
||||
VideoPathManager.Instance.getFSOriginalVideoFilePath(source.keptOriginalFilename)
|
||||
)
|
||||
|
||||
const oldFileUrl = source.fileUrl
|
||||
|
||||
source.fileUrl = null
|
||||
source.storage = FileStorage.FILE_SYSTEM
|
||||
await source.save()
|
||||
|
||||
logger.debug('Removing original video file %s because it\'s now on file system', oldFileUrl, lTagsBase())
|
||||
|
||||
await removeOriginalFileObjectStorage(source)
|
||||
}
|
||||
|
||||
async function moveWebVideoFiles (video: MVideoWithAllFiles) {
|
||||
for (const file of video.VideoFiles) {
|
||||
if (file.storage === FileStorage.FILE_SYSTEM) continue
|
||||
|
||||
await makeWebVideoFileAvailable(file.filename, VideoPathManager.Instance.getFSVideoFileOutputPath(video, file))
|
||||
await onFileMoved({
|
||||
await onVideoFileMoved({
|
||||
videoOrPlaylist: video,
|
||||
file,
|
||||
objetStorageRemover: () => removeWebVideoObjectStorage(file)
|
||||
@ -75,7 +99,7 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
|
||||
await makeHLSFileAvailable(playlistWithVideo, playlistFilename, join(getHLSDirectory(video), playlistFilename))
|
||||
await makeHLSFileAvailable(playlistWithVideo, file.filename, join(getHLSDirectory(video), file.filename))
|
||||
|
||||
await onFileMoved({
|
||||
await onVideoFileMoved({
|
||||
videoOrPlaylist: playlistWithVideo,
|
||||
file,
|
||||
objetStorageRemover: async () => {
|
||||
@ -87,7 +111,7 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
|
||||
}
|
||||
}
|
||||
|
||||
async function onFileMoved (options: {
|
||||
async function onVideoFileMoved (options: {
|
||||
videoOrPlaylist: MVideo | MStreamingPlaylistVideo
|
||||
file: MVideoFile
|
||||
objetStorageRemover: () => Promise<any>
|
||||
|
@ -1,15 +1,16 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { remove } from 'fs-extra/esm'
|
||||
import { join } from 'path'
|
||||
import { MoveStoragePayload, VideoStateType, FileStorage } from '@peertube/peertube-models'
|
||||
import { FileStorage, MoveStoragePayload, VideoStateType } from '@peertube/peertube-models'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { updateTorrentMetadata } from '@server/helpers/webtorrent.js'
|
||||
import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
|
||||
import { storeHLSFileFromFilename, storeWebVideoFile } from '@server/lib/object-storage/index.js'
|
||||
import { storeHLSFileFromFilename, storeOriginalVideoFile, storeWebVideoFile } from '@server/lib/object-storage/index.js'
|
||||
import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state.js'
|
||||
import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
|
||||
import { MVideoSource } from '@server/types/models/video/video-source.js'
|
||||
import { Job } from 'bullmq'
|
||||
import { remove } from 'fs-extra/esm'
|
||||
import { join } from 'path'
|
||||
import { moveToJob, onMoveToStorageFailure } from './shared/move-video.js'
|
||||
|
||||
const lTagsBase = loggerTagsFactory('move-object-storage')
|
||||
@ -25,6 +26,7 @@ export async function processMoveToObjectStorage (job: Job) {
|
||||
|
||||
moveWebVideoFiles,
|
||||
moveHLSFiles,
|
||||
moveVideoSourceFile,
|
||||
doAfterLastMove: video => doAfterLastMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }),
|
||||
moveToFailedState: moveToFailedMoveToObjectStorageState
|
||||
})
|
||||
@ -43,6 +45,21 @@ export async function onMoveToObjectStorageFailure (job: Job, err: any) {
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function moveVideoSourceFile (source: MVideoSource) {
|
||||
if (source.storage !== FileStorage.FILE_SYSTEM) return
|
||||
|
||||
const sourcePath = VideoPathManager.Instance.getFSOriginalVideoFilePath(source.keptOriginalFilename)
|
||||
const fileUrl = await storeOriginalVideoFile(sourcePath, source.keptOriginalFilename)
|
||||
|
||||
source.storage = FileStorage.OBJECT_STORAGE
|
||||
source.fileUrl = fileUrl
|
||||
await source.save()
|
||||
|
||||
logger.debug('Removing original video file ' + sourcePath + ' because it\'s now on object storage', lTagsBase())
|
||||
|
||||
await remove(sourcePath)
|
||||
}
|
||||
|
||||
async function moveWebVideoFiles (video: MVideoWithAllFiles) {
|
||||
for (const file of video.VideoFiles) {
|
||||
if (file.storage !== FileStorage.FILE_SYSTEM) continue
|
||||
@ -50,7 +67,7 @@ async function moveWebVideoFiles (video: MVideoWithAllFiles) {
|
||||
const fileUrl = await storeWebVideoFile(video, file)
|
||||
|
||||
const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file)
|
||||
await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
|
||||
await onVideoFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
|
||||
}
|
||||
}
|
||||
|
||||
@ -70,12 +87,12 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
|
||||
|
||||
const oldPath = join(getHLSDirectory(video), file.filename)
|
||||
|
||||
await onFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath })
|
||||
await onVideoFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function onFileMoved (options: {
|
||||
async function onVideoFileMoved (options: {
|
||||
videoOrPlaylist: MVideo | MStreamingPlaylistVideo
|
||||
file: MVideoFile
|
||||
fileUrl: string
|
||||
|
@ -1,8 +1,10 @@
|
||||
import { LoggerTags, logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
|
||||
import { VideoSourceModel } from '@server/models/video/video-source.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { MVideoWithAllFiles } from '@server/types/models/index.js'
|
||||
import { MVideoSource } from '@server/types/models/video/video-source.js'
|
||||
|
||||
export async function moveToJob (options: {
|
||||
jobId: string
|
||||
@ -11,10 +13,20 @@ export async function moveToJob (options: {
|
||||
|
||||
moveWebVideoFiles: (video: MVideoWithAllFiles) => Promise<void>
|
||||
moveHLSFiles: (video: MVideoWithAllFiles) => Promise<void>
|
||||
moveVideoSourceFile: (source: MVideoSource) => Promise<void>
|
||||
moveToFailedState: (video: MVideoWithAllFiles) => Promise<void>
|
||||
doAfterLastMove: (video: MVideoWithAllFiles) => Promise<void>
|
||||
}) {
|
||||
const { jobId, loggerTags, videoUUID, moveHLSFiles, moveWebVideoFiles, moveToFailedState, doAfterLastMove } = options
|
||||
const {
|
||||
jobId,
|
||||
loggerTags,
|
||||
videoUUID,
|
||||
moveVideoSourceFile,
|
||||
moveHLSFiles,
|
||||
moveWebVideoFiles,
|
||||
moveToFailedState,
|
||||
doAfterLastMove
|
||||
} = options
|
||||
|
||||
const lTagsBase = loggerTagsFactory(...loggerTags)
|
||||
|
||||
@ -31,6 +43,13 @@ export async function moveToJob (options: {
|
||||
const lTags = lTagsBase(video.uuid, video.url)
|
||||
|
||||
try {
|
||||
const source = await VideoSourceModel.loadLatest(video.id)
|
||||
if (source) {
|
||||
logger.debug(`Moving video source ${source.keptOriginalFilename} file of video ${video.uuid}`, lTags)
|
||||
|
||||
await moveVideoSourceFile(source)
|
||||
}
|
||||
|
||||
if (video.VideoFiles) {
|
||||
logger.debug('Moving %d web video files for video %s.', video.VideoFiles.length, video.uuid, lTags)
|
||||
|
||||
|
@ -159,6 +159,20 @@ export async function makeWebVideoFileAvailable (filename: string, destination:
|
||||
return destination
|
||||
}
|
||||
|
||||
export async function makeOriginalFileAvailable (keptOriginalFilename: string, destination: string) {
|
||||
const key = generateOriginalVideoObjectStorageKey(keptOriginalFilename)
|
||||
|
||||
logger.info('Fetching Original Video file %s from object storage to %s.', key, destination, lTags())
|
||||
|
||||
await makeAvailable({
|
||||
key,
|
||||
destination,
|
||||
bucketInfo: CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES
|
||||
})
|
||||
|
||||
return destination
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function getWebVideoFileReadStream (options: {
|
||||
|
Loading…
Reference in New Issue
Block a user