Use private ACL for private videos in s3

This commit is contained in:
Chocobozzz 2022-10-19 10:43:53 +02:00 committed by Chocobozzz
parent 3545e72c68
commit 9ab330b90d
46 changed files with 1753 additions and 845 deletions

View File

@ -46,6 +46,8 @@ jobs:
PGHOST: localhost
NODE_PENDING_JOB_WAIT: 250
ENABLE_OBJECT_STORAGE_TESTS: true
OBJECT_STORAGE_SCALEWAY_KEY_ID: ${{ secrets.OBJECT_STORAGE_SCALEWAY_KEY_ID }}
OBJECT_STORAGE_SCALEWAY_ACCESS_KEY: ${{ secrets.OBJECT_STORAGE_SCALEWAY_ACCESS_KEY }}
steps:
- uses: actions/checkout@v3

View File

@ -148,8 +148,11 @@ object_storage:
region: 'us-east-1'
# Set this ACL on each uploaded object
upload_acl: 'public-read'
upload_acl:
# Set this ACL on each uploaded object of public/unlisted videos
public: 'public-read'
# Set this ACL on each uploaded object of private/internal videos
private: 'private'
credentials:
# You can also use AWS_ACCESS_KEY_ID env variable

View File

@ -146,8 +146,11 @@ object_storage:
region: 'us-east-1'
# Set this ACL on each uploaded object
upload_acl: 'public-read'
upload_acl:
# Set this ACL on each uploaded object of public/unlisted videos
public: 'public-read'
# Set this ACL on each uploaded object of private/internal videos
private: 'private'
credentials:
# You can also use AWS_ACCESS_KEY_ID env variable

View File

@ -78,9 +78,9 @@
"jpeg-js": "0.4.4"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.23.0",
"@aws-sdk/lib-storage": "^3.72.0",
"@aws-sdk/node-http-handler": "^3.82.0",
"@aws-sdk/client-s3": "^3.190.0",
"@aws-sdk/lib-storage": "^3.190.0",
"@aws-sdk/node-http-handler": "^3.190.0",
"@babel/parser": "^7.17.8",
"@node-oauth/oauth2-server": "^4.2.0",
"@opentelemetry/api": "^1.1.0",

View File

@ -107,6 +107,7 @@ import {
wellKnownRouter,
lazyStaticRouter,
servicesRouter,
objectStorageProxyRouter,
pluginsRouter,
webfingerRouter,
trackerRouter,
@ -240,6 +241,7 @@ app.use('/', wellKnownRouter)
app.use('/', miscRouter)
app.use('/', downloadRouter)
app.use('/', lazyStaticRouter)
app.use('/', objectStorageProxyRouter)
// Client files, last valid routes!
const cliOptions = cli.opts<{ client: boolean, plugins: boolean }>()

View File

@ -5,6 +5,7 @@ import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache
import { Hooks } from '@server/lib/plugins/hooks'
import { VideoPathManager } from '@server/lib/video-path-manager'
import { MStreamingPlaylist, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
import { addQueryParams } from '@shared/core-utils'
import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@shared/models'
import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants'
import { asyncMiddleware, optionalAuthenticate, videosDownloadValidator } from '../middlewares'
@ -84,7 +85,7 @@ async function downloadVideoFile (req: express.Request, res: express.Response) {
if (!checkAllowResult(res, allowParameters, allowedResult)) return
if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
return res.redirect(videoFile.getObjectStorageUrl())
return redirectToObjectStorage({ req, res, video, file: videoFile })
}
await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), path => {
@ -120,7 +121,7 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response
if (!checkAllowResult(res, allowParameters, allowedResult)) return
if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
return res.redirect(videoFile.getObjectStorageUrl())
return redirectToObjectStorage({ req, res, video, file: videoFile })
}
await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(streamingPlaylist), path => {
@ -174,3 +175,20 @@ function checkAllowResult (res: express.Response, allowParameters: any, result?:
return true
}
function redirectToObjectStorage (options: {
req: express.Request
res: express.Response
video: MVideo
file: MVideoFile
}) {
const { req, res, video, file } = options
const baseUrl = file.getObjectStorageUrl(video)
const url = video.hasPrivateStaticPath() && req.query.videoFileToken
? addQueryParams(baseUrl, { videoFileToken: req.query.videoFileToken })
: baseUrl
return res.redirect(url)
}

View File

@ -1,14 +1,15 @@
export * from './activitypub'
export * from './api'
export * from './bots'
export * from './client'
export * from './download'
export * from './feeds'
export * from './services'
export * from './static'
export * from './lazy-static'
export * from './misc'
export * from './webfinger'
export * from './tracker'
export * from './bots'
export * from './object-storage-proxy'
export * from './plugins'
export * from './services'
export * from './static'
export * from './tracker'
export * from './webfinger'
export * from './well-known'

View File

@ -0,0 +1,78 @@
import cors from 'cors'
import express from 'express'
import { OBJECT_STORAGE_PROXY_PATHS } from '@server/initializers/constants'
import { getHLSFileReadStream, getWebTorrentFileReadStream } from '@server/lib/object-storage'
import {
asyncMiddleware,
ensureCanAccessPrivateVideoHLSFiles,
ensureCanAccessVideoPrivateWebTorrentFiles,
optionalAuthenticate
} from '@server/middlewares'
import { HttpStatusCode } from '@shared/models'
const objectStorageProxyRouter = express.Router()
objectStorageProxyRouter.use(cors())
objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEBSEED + ':filename',
optionalAuthenticate,
asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles),
asyncMiddleware(proxifyWebTorrent)
)
objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:filename',
optionalAuthenticate,
asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles),
asyncMiddleware(proxifyHLS)
)
// ---------------------------------------------------------------------------
export {
objectStorageProxyRouter
}
async function proxifyWebTorrent (req: express.Request, res: express.Response) {
const filename = req.params.filename
try {
const stream = await getWebTorrentFileReadStream({
filename,
rangeHeader: req.header('range')
})
return stream.pipe(res)
} catch (err) {
return handleObjectStorageFailure(res, err)
}
}
async function proxifyHLS (req: express.Request, res: express.Response) {
const playlist = res.locals.videoStreamingPlaylist
const video = res.locals.onlyVideo
const filename = req.params.filename
try {
const stream = await getHLSFileReadStream({
playlist: playlist.withVideo(video),
filename,
rangeHeader: req.header('range')
})
return stream.pipe(res)
} catch (err) {
return handleObjectStorageFailure(res, err)
}
}
function handleObjectStorageFailure (res: express.Response, err: Error) {
if (err.name === 'NoSuchKey') {
return res.sendStatus(HttpStatusCode.NOT_FOUND_404)
}
return res.fail({
status: HttpStatusCode.INTERNAL_SERVER_ERROR_500,
message: err.message,
type: err.name
})
}

View File

@ -165,7 +165,7 @@ function generateMagnetUri (
const xs = videoFile.getTorrentUrl()
const announce = trackerUrls
let urlList = video.requiresAuth(video.uuid)
let urlList = video.hasPrivateStaticPath()
? []
: [ videoFile.getFileUrl(video) ]
@ -243,7 +243,7 @@ function buildAnnounceList () {
}
function buildUrlList (video: MVideo, videoFile: MVideoFile) {
if (video.requiresAuth(video.uuid)) return []
if (video.hasPrivateStaticPath()) return []
return [ videoFile.getFileUrl(video) ]
}

View File

@ -278,6 +278,14 @@ function checkObjectStorageConfig () {
'Object storage bucket prefixes should be set to different values when the same bucket is used for both types of video.'
)
}
if (!CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PUBLIC) {
throw new Error('object_storage.upload_acl.public must be set')
}
if (!CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PRIVATE) {
throw new Error('object_storage.upload_acl.private must be set')
}
}
}

View File

@ -118,7 +118,10 @@ const CONFIG = {
MAX_UPLOAD_PART: bytes.parse(config.get<string>('object_storage.max_upload_part')),
ENDPOINT: config.get<string>('object_storage.endpoint'),
REGION: config.get<string>('object_storage.region'),
UPLOAD_ACL: config.get<string>('object_storage.upload_acl'),
UPLOAD_ACL: {
PUBLIC: config.get<string>('object_storage.upload_acl.public'),
PRIVATE: config.get<string>('object_storage.upload_acl.private')
},
CREDENTIALS: {
ACCESS_KEY_ID: config.get<string>('object_storage.credentials.access_key_id'),
SECRET_ACCESS_KEY: config.get<string>('object_storage.credentials.secret_access_key')

View File

@ -685,6 +685,13 @@ const LAZY_STATIC_PATHS = {
VIDEO_CAPTIONS: '/lazy-static/video-captions/',
TORRENTS: '/lazy-static/torrents/'
}
const OBJECT_STORAGE_PROXY_PATHS = {
PRIVATE_WEBSEED: '/object-storage-proxy/webseed/private/',
STREAMING_PLAYLISTS: {
PRIVATE_HLS: '/object-storage-proxy/streaming-playlists/hls/private/'
}
}
// Cache control
const STATIC_MAX_AGE = {
@ -995,6 +1002,7 @@ export {
VIDEO_LIVE,
PEERTUBE_VERSION,
LAZY_STATIC_PATHS,
OBJECT_STORAGE_PROXY_PATHS,
SEARCH_INDEX,
DIRECTORIES,
RESUMABLE_UPLOAD_SESSION_LIFETIME,

View File

@ -5,6 +5,7 @@ import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { MStreamingPlaylistVideo } from '@server/types/models'
import { buildSha256Segment } from '../hls'
import { storeHLSFileFromPath } from '../object-storage'
import PQueue from 'p-queue'
const lTags = loggerTagsFactory('live')
@ -16,6 +17,7 @@ class LiveSegmentShaStore {
private readonly sha256Path: string
private readonly streamingPlaylist: MStreamingPlaylistVideo
private readonly sendToObjectStorage: boolean
private readonly writeQueue = new PQueue({ concurrency: 1 })
constructor (options: {
videoUUID: string
@ -37,7 +39,11 @@ class LiveSegmentShaStore {
const segmentName = basename(segmentPath)
this.segmentsSha256.set(segmentName, shaResult)
await this.writeToDisk()
try {
await this.writeToDisk()
} catch (err) {
logger.error('Cannot write sha segments to disk.', { err })
}
}
async removeSegmentSha (segmentPath: string) {
@ -55,19 +61,20 @@ class LiveSegmentShaStore {
await this.writeToDisk()
}
private async writeToDisk () {
await writeJson(this.sha256Path, mapToJSON(this.segmentsSha256))
private writeToDisk () {
return this.writeQueue.add(async () => {
await writeJson(this.sha256Path, mapToJSON(this.segmentsSha256))
if (this.sendToObjectStorage) {
const url = await storeHLSFileFromPath(this.streamingPlaylist, this.sha256Path)
if (this.sendToObjectStorage) {
const url = await storeHLSFileFromPath(this.streamingPlaylist, this.sha256Path)
if (this.streamingPlaylist.segmentsSha256Url !== url) {
this.streamingPlaylist.segmentsSha256Url = url
await this.streamingPlaylist.save()
if (this.streamingPlaylist.segmentsSha256Url !== url) {
this.streamingPlaylist.segmentsSha256Url = url
await this.streamingPlaylist.save()
}
}
}
})
}
}
export {

View File

@ -2,18 +2,21 @@ import { createReadStream, createWriteStream, ensureDir, ReadStream } from 'fs-e
import { dirname } from 'path'
import { Readable } from 'stream'
import {
_Object,
CompleteMultipartUploadCommandOutput,
DeleteObjectCommand,
GetObjectCommand,
ListObjectsV2Command,
PutObjectCommandInput
PutObjectAclCommand,
PutObjectCommandInput,
S3Client
} from '@aws-sdk/client-s3'
import { Upload } from '@aws-sdk/lib-storage'
import { pipelinePromise } from '@server/helpers/core-utils'
import { isArray } from '@server/helpers/custom-validators/misc'
import { logger } from '@server/helpers/logger'
import { CONFIG } from '@server/initializers/config'
import { getPrivateUrl } from '../urls'
import { getInternalUrl } from '../urls'
import { getClient } from './client'
import { lTags } from './logger'
@ -44,69 +47,91 @@ async function storeObject (options: {
inputPath: string
objectStorageKey: string
bucketInfo: BucketInfo
isPrivate: boolean
}): Promise<string> {
const { inputPath, objectStorageKey, bucketInfo } = options
const { inputPath, objectStorageKey, bucketInfo, isPrivate } = options
logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags())
const fileStream = createReadStream(inputPath)
return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo })
return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo, isPrivate })
}
// ---------------------------------------------------------------------------
async function removeObject (filename: string, bucketInfo: BucketInfo) {
const command = new DeleteObjectCommand({
function updateObjectACL (options: {
objectStorageKey: string
bucketInfo: BucketInfo
isPrivate: boolean
}) {
const { objectStorageKey, bucketInfo, isPrivate } = options
const key = buildKey(objectStorageKey, bucketInfo)
logger.debug('Updating ACL file %s in bucket %s', key, bucketInfo.BUCKET_NAME, lTags())
const command = new PutObjectAclCommand({
Bucket: bucketInfo.BUCKET_NAME,
Key: buildKey(filename, bucketInfo)
Key: key,
ACL: getACL(isPrivate)
})
return getClient().send(command)
}
async function removePrefix (prefix: string, bucketInfo: BucketInfo) {
const s3Client = getClient()
function updatePrefixACL (options: {
prefix: string
bucketInfo: BucketInfo
isPrivate: boolean
}) {
const { prefix, bucketInfo, isPrivate } = options
const commandPrefix = bucketInfo.PREFIX + prefix
const listCommand = new ListObjectsV2Command({
logger.debug('Updating ACL of files in prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags())
return applyOnPrefix({
prefix,
bucketInfo,
commandBuilder: obj => {
return new PutObjectAclCommand({
Bucket: bucketInfo.BUCKET_NAME,
Key: obj.Key,
ACL: getACL(isPrivate)
})
}
})
}
// ---------------------------------------------------------------------------
function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) {
const key = buildKey(objectStorageKey, bucketInfo)
logger.debug('Removing file %s in bucket %s', key, bucketInfo.BUCKET_NAME, lTags())
const command = new DeleteObjectCommand({
Bucket: bucketInfo.BUCKET_NAME,
Prefix: commandPrefix
Key: key
})
const listedObjects = await s3Client.send(listCommand)
return getClient().send(command)
}
function removePrefix (prefix: string, bucketInfo: BucketInfo) {
// FIXME: use bulk delete when s3ninja will support this operation
// const deleteParams = {
// Bucket: bucketInfo.BUCKET_NAME,
// Delete: { Objects: [] }
// }
if (isArray(listedObjects.Contents) !== true) {
const message = `Cannot remove ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.`
logger.debug('Removing prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags())
logger.error(message, { response: listedObjects, ...lTags() })
throw new Error(message)
}
for (const object of listedObjects.Contents) {
const command = new DeleteObjectCommand({
Bucket: bucketInfo.BUCKET_NAME,
Key: object.Key
})
await s3Client.send(command)
// FIXME: use bulk delete when s3ninja will support this operation
// deleteParams.Delete.Objects.push({ Key: object.Key })
}
// FIXME: use bulk delete when s3ninja will support this operation
// const deleteCommand = new DeleteObjectsCommand(deleteParams)
// await s3Client.send(deleteCommand)
// Repeat if not all objects could be listed at once (limit of 1000?)
if (listedObjects.IsTruncated) await removePrefix(prefix, bucketInfo)
return applyOnPrefix({
prefix,
bucketInfo,
commandBuilder: obj => {
return new DeleteObjectCommand({
Bucket: bucketInfo.BUCKET_NAME,
Key: obj.Key
})
}
})
}
// ---------------------------------------------------------------------------
@ -138,14 +163,42 @@ function buildKey (key: string, bucketInfo: BucketInfo) {
// ---------------------------------------------------------------------------
async function createObjectReadStream (options: {
key: string
bucketInfo: BucketInfo
rangeHeader: string
}) {
const { key, bucketInfo, rangeHeader } = options
const command = new GetObjectCommand({
Bucket: bucketInfo.BUCKET_NAME,
Key: buildKey(key, bucketInfo),
Range: rangeHeader
})
const response = await getClient().send(command)
return response.Body as Readable
}
// ---------------------------------------------------------------------------
export {
BucketInfo,
buildKey,
storeObject,
removeObject,
removePrefix,
makeAvailable,
listKeysOfPrefix
updateObjectACL,
updatePrefixACL,
listKeysOfPrefix,
createObjectReadStream
}
// ---------------------------------------------------------------------------
@ -154,17 +207,15 @@ async function uploadToStorage (options: {
content: ReadStream
objectStorageKey: string
bucketInfo: BucketInfo
isPrivate: boolean
}) {
const { content, objectStorageKey, bucketInfo } = options
const { content, objectStorageKey, bucketInfo, isPrivate } = options
const input: PutObjectCommandInput = {
Body: content,
Bucket: bucketInfo.BUCKET_NAME,
Key: buildKey(objectStorageKey, bucketInfo)
}
if (CONFIG.OBJECT_STORAGE.UPLOAD_ACL) {
input.ACL = CONFIG.OBJECT_STORAGE.UPLOAD_ACL
Key: buildKey(objectStorageKey, bucketInfo),
ACL: getACL(isPrivate)
}
const parallelUploads3 = new Upload({
@ -194,5 +245,50 @@ async function uploadToStorage (options: {
bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()
)
return getPrivateUrl(bucketInfo, objectStorageKey)
return getInternalUrl(bucketInfo, objectStorageKey)
}
async function applyOnPrefix (options: {
prefix: string
bucketInfo: BucketInfo
commandBuilder: (obj: _Object) => Parameters<S3Client['send']>[0]
continuationToken?: string
}) {
const { prefix, bucketInfo, commandBuilder, continuationToken } = options
const s3Client = getClient()
const commandPrefix = bucketInfo.PREFIX + prefix
const listCommand = new ListObjectsV2Command({
Bucket: bucketInfo.BUCKET_NAME,
Prefix: commandPrefix,
ContinuationToken: continuationToken
})
const listedObjects = await s3Client.send(listCommand)
if (isArray(listedObjects.Contents) !== true) {
const message = `Cannot apply function on ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.`
logger.error(message, { response: listedObjects, ...lTags() })
throw new Error(message)
}
for (const object of listedObjects.Contents) {
const command = commandBuilder(object)
await s3Client.send(command)
}
// Repeat if not all objects could be listed at once (limit of 1000?)
if (listedObjects.IsTruncated) {
await applyOnPrefix({ ...options, continuationToken: listedObjects.ContinuationToken })
}
}
function getACL (isPrivate: boolean) {
return isPrivate
? CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PRIVATE
: CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PUBLIC
}

View File

@ -1,10 +1,14 @@
import { CONFIG } from '@server/initializers/config'
import { OBJECT_STORAGE_PROXY_PATHS, WEBSERVER } from '@server/initializers/constants'
import { MVideoUUID } from '@server/types/models'
import { BucketInfo, buildKey, getEndpointParsed } from './shared'
function getPrivateUrl (config: BucketInfo, keyWithoutPrefix: string) {
function getInternalUrl (config: BucketInfo, keyWithoutPrefix: string) {
return getBaseUrl(config) + buildKey(keyWithoutPrefix, config)
}
// ---------------------------------------------------------------------------
function getWebTorrentPublicFileUrl (fileUrl: string) {
const baseUrl = CONFIG.OBJECT_STORAGE.VIDEOS.BASE_URL
if (!baseUrl) return fileUrl
@ -19,11 +23,28 @@ function getHLSPublicFileUrl (fileUrl: string) {
return replaceByBaseUrl(fileUrl, baseUrl)
}
// ---------------------------------------------------------------------------
function getHLSPrivateFileUrl (video: MVideoUUID, filename: string) {
return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + video.uuid + `/${filename}`
}
function getWebTorrentPrivateFileUrl (filename: string) {
return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEBSEED + filename
}
// ---------------------------------------------------------------------------
export {
getPrivateUrl,
getInternalUrl,
getWebTorrentPublicFileUrl,
replaceByBaseUrl,
getHLSPublicFileUrl
getHLSPublicFileUrl,
getHLSPrivateFileUrl,
getWebTorrentPrivateFileUrl,
replaceByBaseUrl
}
// ---------------------------------------------------------------------------

View File

@ -5,7 +5,17 @@ import { MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/types/model
import { getHLSDirectory } from '../paths'
import { VideoPathManager } from '../video-path-manager'
import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys'
import { listKeysOfPrefix, lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared'
import {
createObjectReadStream,
listKeysOfPrefix,
lTags,
makeAvailable,
removeObject,
removePrefix,
storeObject,
updateObjectACL,
updatePrefixACL
} from './shared'
function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) {
return listKeysOfPrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
@ -17,7 +27,8 @@ function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename:
return storeObject({
inputPath: join(getHLSDirectory(playlist.Video), filename),
objectStorageKey: generateHLSObjectStorageKey(playlist, filename),
bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
isPrivate: playlist.Video.hasPrivateStaticPath()
})
}
@ -25,7 +36,8 @@ function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string)
return storeObject({
inputPath: path,
objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)),
bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
isPrivate: playlist.Video.hasPrivateStaticPath()
})
}
@ -35,7 +47,26 @@ function storeWebTorrentFile (video: MVideo, file: MVideoFile) {
return storeObject({
inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file),
objectStorageKey: generateWebTorrentObjectStorageKey(file.filename),
bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS
bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS,
isPrivate: video.hasPrivateStaticPath()
})
}
// ---------------------------------------------------------------------------
function updateWebTorrentFileACL (video: MVideo, file: MVideoFile) {
return updateObjectACL({
objectStorageKey: generateWebTorrentObjectStorageKey(file.filename),
bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS,
isPrivate: video.hasPrivateStaticPath()
})
}
function updateHLSFilesACL (playlist: MStreamingPlaylistVideo) {
return updatePrefixACL({
prefix: generateHLSObjectBaseStorageKey(playlist),
bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
isPrivate: playlist.Video.hasPrivateStaticPath()
})
}
@ -87,6 +118,39 @@ async function makeWebTorrentFileAvailable (filename: string, destination: strin
// ---------------------------------------------------------------------------
function getWebTorrentFileReadStream (options: {
filename: string
rangeHeader: string
}) {
const { filename, rangeHeader } = options
const key = generateWebTorrentObjectStorageKey(filename)
return createObjectReadStream({
key,
bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS,
rangeHeader
})
}
function getHLSFileReadStream (options: {
playlist: MStreamingPlaylistVideo
filename: string
rangeHeader: string
}) {
const { playlist, filename, rangeHeader } = options
const key = generateHLSObjectStorageKey(playlist, filename)
return createObjectReadStream({
key,
bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
rangeHeader
})
}
// ---------------------------------------------------------------------------
export {
listHLSFileKeysOf,
@ -94,10 +158,16 @@ export {
storeHLSFileFromFilename,
storeHLSFileFromPath,
updateWebTorrentFileACL,
updateHLSFilesACL,
removeHLSObjectStorage,
removeHLSFileObjectStorage,
removeWebTorrentObjectStorage,
makeWebTorrentFileAvailable,
makeHLSFileAvailable
makeHLSFileAvailable,
getWebTorrentFileReadStream,
getHLSFileReadStream
}

View File

@ -2,8 +2,9 @@ import { move } from 'fs-extra'
import { join } from 'path'
import { logger } from '@server/helpers/logger'
import { DIRECTORIES } from '@server/initializers/constants'
import { MVideo, MVideoFullLight } from '@server/types/models'
import { VideoPrivacy } from '@shared/models'
import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
import { VideoPrivacy, VideoStorage } from '@shared/models'
import { updateHLSFilesACL, updateWebTorrentFileACL } from './object-storage'
function setVideoPrivacy (video: MVideo, newPrivacy: VideoPrivacy) {
if (video.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) {
@ -50,47 +51,77 @@ export {
// ---------------------------------------------------------------------------
type MoveType = 'private-to-public' | 'public-to-private'
async function moveFiles (options: {
type: 'private-to-public' | 'public-to-private'
type: MoveType
video: MVideoFullLight
}) {
const { type, video } = options
const directories = type === 'private-to-public'
? {
webtorrent: { old: DIRECTORIES.VIDEOS.PRIVATE, new: DIRECTORIES.VIDEOS.PUBLIC },
hls: { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC }
}
: {
webtorrent: { old: DIRECTORIES.VIDEOS.PUBLIC, new: DIRECTORIES.VIDEOS.PRIVATE },
hls: { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE }
}
for (const file of video.VideoFiles) {
const source = join(directories.webtorrent.old, file.filename)
const destination = join(directories.webtorrent.new, file.filename)
try {
logger.info('Moving WebTorrent files of %s after privacy change (%s -> %s).', video.uuid, source, destination)
await move(source, destination)
} catch (err) {
logger.error('Cannot move webtorrent file %s to %s after privacy change', source, destination, { err })
if (file.storage === VideoStorage.FILE_SYSTEM) {
await moveWebTorrentFileOnFS(type, video, file)
} else {
await updateWebTorrentFileACL(video, file)
}
}
const hls = video.getHLSPlaylist()
if (hls) {
const source = join(directories.hls.old, video.uuid)
const destination = join(directories.hls.new, video.uuid)
try {
logger.info('Moving HLS files of %s after privacy change (%s -> %s).', video.uuid, source, destination)
await move(source, destination)
} catch (err) {
logger.error('Cannot move HLS file %s to %s after privacy change', source, destination, { err })
if (hls.storage === VideoStorage.FILE_SYSTEM) {
await moveHLSFilesOnFS(type, video)
} else {
await updateHLSFilesACL(hls)
}
}
}
async function moveWebTorrentFileOnFS (type: MoveType, video: MVideo, file: MVideoFile) {
const directories = getWebTorrentDirectories(type)
const source = join(directories.old, file.filename)
const destination = join(directories.new, file.filename)
try {
logger.info('Moving WebTorrent files of %s after privacy change (%s -> %s).', video.uuid, source, destination)
await move(source, destination)
} catch (err) {
logger.error('Cannot move webtorrent file %s to %s after privacy change', source, destination, { err })
}
}
function getWebTorrentDirectories (moveType: MoveType) {
if (moveType === 'private-to-public') {
return { old: DIRECTORIES.VIDEOS.PRIVATE, new: DIRECTORIES.VIDEOS.PUBLIC }
}
return { old: DIRECTORIES.VIDEOS.PUBLIC, new: DIRECTORIES.VIDEOS.PRIVATE }
}
// ---------------------------------------------------------------------------
async function moveHLSFilesOnFS (type: MoveType, video: MVideo) {
const directories = getHLSDirectories(type)
const source = join(directories.old, video.uuid)
const destination = join(directories.new, video.uuid)
try {
logger.info('Moving HLS files of %s after privacy change (%s -> %s).', video.uuid, source, destination)
await move(source, destination)
} catch (err) {
logger.error('Cannot move HLS file %s to %s after privacy change', source, destination, { err })
}
}
function getHLSDirectories (moveType: MoveType) {
if (moveType === 'private-to-public') {
return { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC }
}
return { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE }
}

View File

@ -111,7 +111,7 @@ async function checkCanSeeVideo (options: {
}) {
const { req, res, video, paramId } = options
if (video.requiresAuth(paramId)) {
if (video.requiresAuth({ urlParamId: paramId, checkBlacklist: true })) {
return checkCanSeeAuthVideo(req, res, video)
}
@ -174,13 +174,13 @@ async function checkCanAccessVideoStaticFiles (options: {
res: Response
paramId: string
}) {
const { video, req, res, paramId } = options
const { video, req, res } = options
if (res.locals.oauth?.token.User) {
return checkCanSeeVideo(options)
}
if (!video.requiresAuth(paramId)) return true
if (!video.hasPrivateStaticPath()) return true
const videoFileToken = req.query.videoFileToken
if (!videoFileToken) {

View File

@ -7,10 +7,17 @@ import { logger } from '@server/helpers/logger'
import { LRU_CACHE } from '@server/initializers/constants'
import { VideoModel } from '@server/models/video/video'
import { VideoFileModel } from '@server/models/video/video-file'
import { MStreamingPlaylist, MVideoFile, MVideoThumbnail } from '@server/types/models'
import { HttpStatusCode } from '@shared/models'
import { areValidationErrors, checkCanAccessVideoStaticFiles } from './shared'
const staticFileTokenBypass = new LRUCache<string, boolean>({
type LRUValue = {
allowed: boolean
video?: MVideoThumbnail
file?: MVideoFile
playlist?: MStreamingPlaylist }
const staticFileTokenBypass = new LRUCache<string, LRUValue>({
max: LRU_CACHE.STATIC_VIDEO_FILES_RIGHTS_CHECK.MAX_SIZE,
ttl: LRU_CACHE.STATIC_VIDEO_FILES_RIGHTS_CHECK.TTL
})
@ -27,18 +34,26 @@ const ensureCanAccessVideoPrivateWebTorrentFiles = [
const cacheKey = token + '-' + req.originalUrl
if (staticFileTokenBypass.has(cacheKey)) {
const allowedFromCache = staticFileTokenBypass.get(cacheKey)
const { allowed, file, video } = staticFileTokenBypass.get(cacheKey)
if (allowedFromCache === true) return next()
if (allowed === true) {
res.locals.onlyVideo = video
res.locals.videoFile = file
return next()
}
return res.sendStatus(HttpStatusCode.FORBIDDEN_403)
}
const allowed = await isWebTorrentAllowed(req, res)
const result = await isWebTorrentAllowed(req, res)
staticFileTokenBypass.set(cacheKey, allowed)
staticFileTokenBypass.set(cacheKey, result)
if (allowed !== true) return
if (result.allowed !== true) return
res.locals.onlyVideo = result.video
res.locals.videoFile = result.file
return next()
}
@ -64,18 +79,28 @@ const ensureCanAccessPrivateVideoHLSFiles = [
const cacheKey = token + '-' + videoUUID
if (staticFileTokenBypass.has(cacheKey)) {
const allowedFromCache = staticFileTokenBypass.get(cacheKey)
const { allowed, file, playlist, video } = staticFileTokenBypass.get(cacheKey)
if (allowedFromCache === true) return next()
if (allowed === true) {
res.locals.onlyVideo = video
res.locals.videoFile = file
res.locals.videoStreamingPlaylist = playlist
return next()
}
return res.sendStatus(HttpStatusCode.FORBIDDEN_403)
}
const allowed = await isHLSAllowed(req, res, videoUUID)
const result = await isHLSAllowed(req, res, videoUUID)
staticFileTokenBypass.set(cacheKey, allowed)
staticFileTokenBypass.set(cacheKey, result)
if (allowed !== true) return
if (result.allowed !== true) return
res.locals.onlyVideo = result.video
res.locals.videoFile = result.file
res.locals.videoStreamingPlaylist = result.playlist
return next()
}
@ -96,25 +121,38 @@ async function isWebTorrentAllowed (req: express.Request, res: express.Response)
logger.debug('Unknown static file %s to serve', req.originalUrl, { filename })
res.sendStatus(HttpStatusCode.FORBIDDEN_403)
return false
return { allowed: false }
}
const video = file.getVideo()
const video = await VideoModel.load(file.getVideo().id)
return checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid })
return {
file,
video,
allowed: await checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid })
}
}
async function isHLSAllowed (req: express.Request, res: express.Response, videoUUID: string) {
const video = await VideoModel.load(videoUUID)
const filename = basename(req.path)
const video = await VideoModel.loadWithFiles(videoUUID)
if (!video) {
logger.debug('Unknown static file %s to serve', req.originalUrl, { videoUUID })
res.sendStatus(HttpStatusCode.FORBIDDEN_403)
return false
return { allowed: false }
}
return checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid })
const file = await VideoFileModel.loadByFilename(filename)
return {
file,
video,
playlist: video.getHLSPlaylist(),
allowed: await checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid })
}
}
function extractTokenOrDie (req: express.Request, res: express.Response) {

View File

@ -22,7 +22,12 @@ import validator from 'validator'
import { logger } from '@server/helpers/logger'
import { extractVideo } from '@server/helpers/video'
import { buildRemoteVideoBaseUrl } from '@server/lib/activitypub/url'
import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage'
import {
getHLSPrivateFileUrl,
getHLSPublicFileUrl,
getWebTorrentPrivateFileUrl,
getWebTorrentPublicFileUrl
} from '@server/lib/object-storage'
import { getFSTorrentFilePath } from '@server/lib/paths'
import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
@ -503,7 +508,25 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
return !!this.videoStreamingPlaylistId
}
getObjectStorageUrl () {
// ---------------------------------------------------------------------------
getObjectStorageUrl (video: MVideo) {
if (video.hasPrivateStaticPath()) {
return this.getPrivateObjectStorageUrl(video)
}
return this.getPublicObjectStorageUrl()
}
private getPrivateObjectStorageUrl (video: MVideo) {
if (this.isHLS()) {
return getHLSPrivateFileUrl(video, this.filename)
}
return getWebTorrentPrivateFileUrl(this.filename)
}
private getPublicObjectStorageUrl () {
if (this.isHLS()) {
return getHLSPublicFileUrl(this.fileUrl)
}
@ -511,26 +534,29 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
return getWebTorrentPublicFileUrl(this.fileUrl)
}
getFileUrl (video: MVideo) {
if (this.storage === VideoStorage.OBJECT_STORAGE) {
return this.getObjectStorageUrl()
}
// ---------------------------------------------------------------------------
if (!this.Video) this.Video = video as VideoModel
if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video)
getFileUrl (video: MVideo) {
if (video.isOwned()) {
if (this.storage === VideoStorage.OBJECT_STORAGE) {
return this.getObjectStorageUrl(video)
}
return WEBSERVER.URL + this.getFileStaticPath(video)
}
return this.fileUrl
}
// ---------------------------------------------------------------------------
getFileStaticPath (video: MVideo) {
if (this.isHLS()) {
if (isVideoInPrivateDirectory(video.privacy)) {
return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.filename)
}
if (this.isHLS()) return this.getHLSFileStaticPath(video)
return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename)
}
return this.getWebTorrentFileStaticPath(video)
}
private getWebTorrentFileStaticPath (video: MVideo) {
if (isVideoInPrivateDirectory(video.privacy)) {
return join(STATIC_PATHS.PRIVATE_WEBSEED, this.filename)
}
@ -538,6 +564,16 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
return join(STATIC_PATHS.WEBSEED, this.filename)
}
private getHLSFileStaticPath (video: MVideo) {
if (isVideoInPrivateDirectory(video.privacy)) {
return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.filename)
}
return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename)
}
// ---------------------------------------------------------------------------
getFileDownloadUrl (video: MVideoWithHost) {
const path = this.isHLS()
? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`)

View File

@ -15,7 +15,7 @@ import {
Table,
UpdatedAt
} from 'sequelize-typescript'
import { getHLSPublicFileUrl } from '@server/lib/object-storage'
import { getHLSPrivateFileUrl, getHLSPublicFileUrl } from '@server/lib/object-storage'
import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '@server/lib/paths'
import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
import { VideoFileModel } from '@server/models/video/video-file'
@ -245,10 +245,12 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
this.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(masterPlaylistUrl, files)
}
// ---------------------------------------------------------------------------
getMasterPlaylistUrl (video: MVideo) {
if (video.isOwned()) {
if (this.storage === VideoStorage.OBJECT_STORAGE) {
return getHLSPublicFileUrl(this.playlistUrl)
return this.getMasterPlaylistObjectStorageUrl(video)
}
return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video)
@ -257,10 +259,20 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
return this.playlistUrl
}
private getMasterPlaylistObjectStorageUrl (video: MVideo) {
if (video.hasPrivateStaticPath()) {
return getHLSPrivateFileUrl(video, this.playlistFilename)
}
return getHLSPublicFileUrl(this.playlistUrl)
}
// ---------------------------------------------------------------------------
getSha256SegmentsUrl (video: MVideo) {
if (video.isOwned()) {
if (this.storage === VideoStorage.OBJECT_STORAGE) {
return getHLSPublicFileUrl(this.segmentsSha256Url)
return this.getSha256SegmentsObjectStorageUrl(video)
}
return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video)
@ -269,6 +281,16 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
return this.segmentsSha256Url
}
private getSha256SegmentsObjectStorageUrl (video: MVideo) {
if (video.hasPrivateStaticPath()) {
return getHLSPrivateFileUrl(video, this.segmentsSha256Filename)
}
return getHLSPublicFileUrl(this.segmentsSha256Url)
}
// ---------------------------------------------------------------------------
getStringType () {
if (this.type === VideoStreamingPlaylistType.HLS) return 'hls'

View File

@ -30,6 +30,7 @@ import { removeHLSFileObjectStorage, removeHLSObjectStorage, removeWebTorrentObj
import { tracer } from '@server/lib/opentelemetry/tracing'
import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
import { VideoPathManager } from '@server/lib/video-path-manager'
import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
import { getServerActor } from '@server/models/application/application'
import { ModelCache } from '@server/models/model-cache'
import { buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils'
@ -1764,9 +1765,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
const playlist = this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
if (!playlist) return undefined
playlist.Video = this
return playlist
return playlist.withVideo(this)
}
setHLSPlaylist (playlist: MStreamingPlaylist) {
@ -1868,16 +1867,39 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
return setAsUpdated('video', this.id, transaction)
}
requiresAuth (paramId: string) {
// ---------------------------------------------------------------------------
requiresAuth (options: {
urlParamId: string
checkBlacklist: boolean
}) {
const { urlParamId, checkBlacklist } = options
if (this.privacy === VideoPrivacy.PRIVATE || this.privacy === VideoPrivacy.INTERNAL) {
return true
}
if (this.privacy === VideoPrivacy.UNLISTED) {
if (!isUUIDValid(paramId)) return true
if (urlParamId && !isUUIDValid(urlParamId)) return true
return false
}
return this.privacy === VideoPrivacy.PRIVATE || this.privacy === VideoPrivacy.INTERNAL || !!this.VideoBlacklist
if (checkBlacklist && this.VideoBlacklist) return true
if (this.privacy !== VideoPrivacy.PUBLIC) {
throw new Error(`Unknown video privacy ${this.privacy} to know if the video requires auth`)
}
return false
}
hasPrivateStaticPath () {
return isVideoInPrivateDirectory(this.privacy)
}
// ---------------------------------------------------------------------------
async setNewState (newState: VideoState, isNewVideo: boolean, transaction: Transaction) {
if (this.state === newState) throw new Error('Cannot use same state ' + newState)

View File

@ -1,3 +1,4 @@
export * from './live'
export * from './video-imports'
export * from './video-static-file-privacy'
export * from './videos'

View File

@ -2,7 +2,7 @@
import { expect } from 'chai'
import { expectStartWith, testVideoResolutions } from '@server/tests/shared'
import { areObjectStorageTestsDisabled } from '@shared/core-utils'
import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
import { HttpStatusCode, LiveVideoCreate, VideoPrivacy } from '@shared/models'
import {
createMultipleServers,
@ -46,7 +46,7 @@ async function checkFilesExist (servers: PeerTubeServer[], videoUUID: string, nu
expect(files).to.have.lengthOf(numberOfFiles)
for (const file of files) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
@ -75,16 +75,16 @@ async function checkFilesCleanup (server: PeerTubeServer, videoUUID: string, res
}
describe('Object storage for lives', function () {
if (areObjectStorageTestsDisabled()) return
if (areMockObjectStorageTestsDisabled()) return
let servers: PeerTubeServer[]
before(async function () {
this.timeout(120000)
await ObjectStorageCommand.prepareDefaultBuckets()
await ObjectStorageCommand.prepareDefaultMockBuckets()
servers = await createMultipleServers(2, ObjectStorageCommand.getDefaultConfig())
servers = await createMultipleServers(2, ObjectStorageCommand.getDefaultMockConfig())
await setAccessTokensToServers(servers)
await setDefaultVideoChannel(servers)

View File

@ -2,7 +2,7 @@
import { expect } from 'chai'
import { expectStartWith, FIXTURE_URLS } from '@server/tests/shared'
import { areObjectStorageTestsDisabled } from '@shared/core-utils'
import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
import { HttpStatusCode, VideoPrivacy } from '@shared/models'
import {
createSingleServer,
@ -29,16 +29,16 @@ async function importVideo (server: PeerTubeServer) {
}
describe('Object storage for video import', function () {
if (areObjectStorageTestsDisabled()) return
if (areMockObjectStorageTestsDisabled()) return
let server: PeerTubeServer
before(async function () {
this.timeout(120000)
await ObjectStorageCommand.prepareDefaultBuckets()
await ObjectStorageCommand.prepareDefaultMockBuckets()
server = await createSingleServer(1, ObjectStorageCommand.getDefaultConfig())
server = await createSingleServer(1, ObjectStorageCommand.getDefaultMockConfig())
await setAccessTokensToServers([ server ])
await setDefaultVideoChannel([ server ])
@ -64,7 +64,7 @@ describe('Object storage for video import', function () {
expect(video.streamingPlaylists).to.have.lengthOf(0)
const fileUrl = video.files[0].fileUrl
expectStartWith(fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
expectStartWith(fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.OK_200 })
})
@ -89,13 +89,13 @@ describe('Object storage for video import', function () {
expect(video.streamingPlaylists[0].files).to.have.lengthOf(5)
for (const file of video.files) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
expectStartWith(file.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
for (const file of video.streamingPlaylists[0].files) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}

View File

@ -0,0 +1,336 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { expect } from 'chai'
import { basename } from 'path'
import { expectStartWith } from '@server/tests/shared'
import { areScalewayObjectStorageTestsDisabled, getAllFiles, getHLS } from '@shared/core-utils'
import { HttpStatusCode, LiveVideo, VideoDetails, VideoPrivacy } from '@shared/models'
import {
cleanupTests,
createSingleServer,
findExternalSavedVideo,
makeRawRequest,
ObjectStorageCommand,
PeerTubeServer,
sendRTMPStream,
setAccessTokensToServers,
setDefaultVideoChannel,
stopFfmpeg,
waitJobs
} from '@shared/server-commands'
describe('Object storage for video static file privacy', function () {
// We need real world object storage to check ACL
if (areScalewayObjectStorageTestsDisabled()) return
let server: PeerTubeServer
let userToken: string
before(async function () {
this.timeout(120000)
server = await createSingleServer(1, ObjectStorageCommand.getDefaultScalewayConfig(1))
await setAccessTokensToServers([ server ])
await setDefaultVideoChannel([ server ])
await server.config.enableMinimumTranscoding()
userToken = await server.users.generateUserAndToken('user1')
})
describe('VOD', function () {
let privateVideoUUID: string
let publicVideoUUID: string
let userPrivateVideoUUID: string
async function checkPrivateFiles (uuid: string) {
const video = await server.videos.getWithToken({ id: uuid })
for (const file of video.files) {
expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/webseed/private/')
await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
}
for (const file of getAllFiles(video)) {
const internalFileUrl = await server.sql.getInternalFileUrl(file.id)
expectStartWith(internalFileUrl, ObjectStorageCommand.getScalewayBaseUrl())
await makeRawRequest({ url: internalFileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
}
const hls = getHLS(video)
if (hls) {
for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) {
expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/')
}
await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
for (const file of hls.files) {
expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/streaming-playlists/hls/private/')
await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
}
}
}
async function checkPublicFiles (uuid: string) {
const video = await server.videos.getWithToken({ id: uuid })
for (const file of getAllFiles(video)) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getScalewayBaseUrl())
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
const hls = getHLS(video)
if (hls) {
expectStartWith(hls.playlistUrl, ObjectStorageCommand.getScalewayBaseUrl())
expectStartWith(hls.segmentsSha256Url, ObjectStorageCommand.getScalewayBaseUrl())
await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
}
}
async function getSampleFileUrls (videoId: string) {
const video = await server.videos.getWithToken({ id: videoId })
return {
webTorrentFile: video.files[0].fileUrl,
hlsFile: getHLS(video).files[0].fileUrl
}
}
it('Should upload a private video and have appropriate object storage ACL', async function () {
this.timeout(60000)
{
const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
privateVideoUUID = uuid
}
{
const { uuid } = await server.videos.quickUpload({ name: 'user video', token: userToken, privacy: VideoPrivacy.PRIVATE })
userPrivateVideoUUID = uuid
}
await waitJobs([ server ])
await checkPrivateFiles(privateVideoUUID)
})
it('Should upload a public video and have appropriate object storage ACL', async function () {
this.timeout(60000)
const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.UNLISTED })
await waitJobs([ server ])
publicVideoUUID = uuid
await checkPublicFiles(publicVideoUUID)
})
it('Should not get files without appropriate OAuth token', async function () {
this.timeout(60000)
const { webTorrentFile, hlsFile } = await getSampleFileUrls(privateVideoUUID)
await makeRawRequest({ url: webTorrentFile, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await makeRawRequest({ url: webTorrentFile, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: hlsFile, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await makeRawRequest({ url: hlsFile, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
})
it('Should not get HLS file of another video', async function () {
this.timeout(60000)
const privateVideo = await server.videos.getWithToken({ id: privateVideoUUID })
const hlsFilename = basename(getHLS(privateVideo).files[0].fileUrl)
const badUrl = server.url + '/object-storage-proxy/streaming-playlists/hls/private/' + userPrivateVideoUUID + '/' + hlsFilename
const goodUrl = server.url + '/object-storage-proxy/streaming-playlists/hls/private/' + privateVideoUUID + '/' + hlsFilename
await makeRawRequest({ url: badUrl, token: server.accessToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
await makeRawRequest({ url: goodUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
})
it('Should correctly check OAuth or video file token', async function () {
this.timeout(60000)
const badVideoFileToken = await server.videoToken.getVideoFileToken({ token: userToken, videoId: userPrivateVideoUUID })
const goodVideoFileToken = await server.videoToken.getVideoFileToken({ videoId: privateVideoUUID })
const { webTorrentFile, hlsFile } = await getSampleFileUrls(privateVideoUUID)
for (const url of [ webTorrentFile, hlsFile ]) {
await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url, query: { videoFileToken: badVideoFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await makeRawRequest({ url, query: { videoFileToken: goodVideoFileToken }, expectedStatus: HttpStatusCode.OK_200 })
}
})
it('Should update public video to private', async function () {
this.timeout(60000)
await server.videos.update({ id: publicVideoUUID, attributes: { privacy: VideoPrivacy.INTERNAL } })
await checkPrivateFiles(publicVideoUUID)
})
it('Should update private video to public', async function () {
this.timeout(60000)
await server.videos.update({ id: publicVideoUUID, attributes: { privacy: VideoPrivacy.PUBLIC } })
await checkPublicFiles(publicVideoUUID)
})
after(async function () {
this.timeout(30000)
if (privateVideoUUID) await server.videos.remove({ id: privateVideoUUID })
if (publicVideoUUID) await server.videos.remove({ id: publicVideoUUID })
if (userPrivateVideoUUID) await server.videos.remove({ id: userPrivateVideoUUID })
await waitJobs([ server ])
})
})
describe('Live', function () {
let normalLiveId: string
let normalLive: LiveVideo
let permanentLiveId: string
let permanentLive: LiveVideo
let unrelatedFileToken: string
async function checkLiveFiles (live: LiveVideo, liveId: string) {
const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey })
await server.live.waitUntilPublished({ videoId: liveId })
const video = await server.videos.getWithToken({ id: liveId })
const fileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid })
const hls = video.streamingPlaylists[0]
for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) {
expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/')
await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
}
await stopFfmpeg(ffmpegCommand)
}
async function checkReplay (replay: VideoDetails) {
const fileToken = await server.videoToken.getVideoFileToken({ videoId: replay.uuid })
const hls = replay.streamingPlaylists[0]
expect(hls.files).to.not.have.lengthOf(0)
for (const file of hls.files) {
await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: file.fileUrl, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: file.fileUrl, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await makeRawRequest({
url: file.fileUrl,
query: { videoFileToken: unrelatedFileToken },
expectedStatus: HttpStatusCode.FORBIDDEN_403
})
}
for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) {
expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/')
await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
}
}
before(async function () {
await server.config.enableMinimumTranscoding()
const { uuid } = await server.videos.quickUpload({ name: 'another video' })
unrelatedFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid })
await server.config.enableLive({
allowReplay: true,
transcoding: true,
resolutions: 'min'
})
{
const { video, live } = await server.live.quickCreate({ saveReplay: true, permanentLive: false, privacy: VideoPrivacy.PRIVATE })
normalLiveId = video.uuid
normalLive = live
}
{
const { video, live } = await server.live.quickCreate({ saveReplay: true, permanentLive: true, privacy: VideoPrivacy.PRIVATE })
permanentLiveId = video.uuid
permanentLive = live
}
})
it('Should create a private normal live and have a private static path', async function () {
this.timeout(240000)
await checkLiveFiles(normalLive, normalLiveId)
})
it('Should create a private permanent live and have a private static path', async function () {
this.timeout(240000)
await checkLiveFiles(permanentLive, permanentLiveId)
})
it('Should have created a replay of the normal live with a private static path', async function () {
this.timeout(240000)
await server.live.waitUntilReplacedByReplay({ videoId: normalLiveId })
const replay = await server.videos.getWithToken({ id: normalLiveId })
await checkReplay(replay)
})
it('Should have created a replay of the permanent live with a private static path', async function () {
this.timeout(240000)
await server.live.waitUntilWaiting({ videoId: permanentLiveId })
await waitJobs([ server ])
const live = await server.videos.getWithToken({ id: permanentLiveId })
const replayFromList = await findExternalSavedVideo(server, live)
const replay = await server.videos.getWithToken({ id: replayFromList.id })
await checkReplay(replay)
})
})
after(async function () {
await cleanupTests([ server ])
})
})

View File

@ -11,7 +11,7 @@ import {
generateHighBitrateVideo,
MockObjectStorage
} from '@server/tests/shared'
import { areObjectStorageTestsDisabled } from '@shared/core-utils'
import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
import { HttpStatusCode, VideoDetails } from '@shared/models'
import {
cleanupTests,
@ -52,7 +52,7 @@ async function checkFiles (options: {
for (const file of video.files) {
const baseUrl = baseMockUrl
? `${baseMockUrl}/${webtorrentBucket}/`
: `http://${webtorrentBucket}.${ObjectStorageCommand.getEndpointHost()}/`
: `http://${webtorrentBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
const prefix = webtorrentPrefix || ''
const start = baseUrl + prefix
@ -73,7 +73,7 @@ async function checkFiles (options: {
const baseUrl = baseMockUrl
? `${baseMockUrl}/${playlistBucket}/`
: `http://${playlistBucket}.${ObjectStorageCommand.getEndpointHost()}/`
: `http://${playlistBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
const prefix = playlistPrefix || ''
const start = baseUrl + prefix
@ -141,16 +141,16 @@ function runTestSuite (options: {
const port = await mockObjectStorage.initialize()
baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined
await ObjectStorageCommand.createBucket(options.playlistBucket)
await ObjectStorageCommand.createBucket(options.webtorrentBucket)
await ObjectStorageCommand.createMockBucket(options.playlistBucket)
await ObjectStorageCommand.createMockBucket(options.webtorrentBucket)
const config = {
object_storage: {
enabled: true,
endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
region: ObjectStorageCommand.getRegion(),
endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
region: ObjectStorageCommand.getMockRegion(),
credentials: ObjectStorageCommand.getCredentialsConfig(),
credentials: ObjectStorageCommand.getMockCredentialsConfig(),
max_upload_part: options.maxUploadPart || '5MB',
@ -261,7 +261,7 @@ function runTestSuite (options: {
}
describe('Object storage for videos', function () {
if (areObjectStorageTestsDisabled()) return
if (areMockObjectStorageTestsDisabled()) return
describe('Test config', function () {
let server: PeerTubeServer
@ -269,17 +269,17 @@ describe('Object storage for videos', function () {
const baseConfig = {
object_storage: {
enabled: true,
endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
region: ObjectStorageCommand.getRegion(),
endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
region: ObjectStorageCommand.getMockRegion(),
credentials: ObjectStorageCommand.getCredentialsConfig(),
credentials: ObjectStorageCommand.getMockCredentialsConfig(),
streaming_playlists: {
bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_BUCKET
bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_MOCK_BUCKET
},
videos: {
bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_BUCKET
bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_MOCK_BUCKET
}
}
}
@ -310,7 +310,7 @@ describe('Object storage for videos', function () {
it('Should fail with bad credentials', async function () {
this.timeout(60000)
await ObjectStorageCommand.prepareDefaultBuckets()
await ObjectStorageCommand.prepareDefaultMockBuckets()
const config = merge({}, baseConfig, {
object_storage: {
@ -334,7 +334,7 @@ describe('Object storage for videos', function () {
it('Should succeed with credentials from env', async function () {
this.timeout(60000)
await ObjectStorageCommand.prepareDefaultBuckets()
await ObjectStorageCommand.prepareDefaultMockBuckets()
const config = merge({}, baseConfig, {
object_storage: {
@ -345,7 +345,7 @@ describe('Object storage for videos', function () {
}
})
const goodCredentials = ObjectStorageCommand.getCredentialsConfig()
const goodCredentials = ObjectStorageCommand.getMockCredentialsConfig()
server = await createSingleServer(1, config, {
env: {
@ -361,7 +361,7 @@ describe('Object storage for videos', function () {
await waitJobs([ server ], true)
const video = await server.videos.get({ id: uuid })
expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
})
after(async function () {

View File

@ -2,7 +2,7 @@
import { expect } from 'chai'
import { expectNotStartWith, expectStartWith, FIXTURE_URLS, MockProxy } from '@server/tests/shared'
import { areObjectStorageTestsDisabled } from '@shared/core-utils'
import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
import { HttpStatusCode, VideoPrivacy } from '@shared/models'
import {
cleanupTests,
@ -120,40 +120,40 @@ describe('Test proxy', function () {
})
describe('Object storage', function () {
if (areObjectStorageTestsDisabled()) return
if (areMockObjectStorageTestsDisabled()) return
before(async function () {
this.timeout(30000)
await ObjectStorageCommand.prepareDefaultBuckets()
await ObjectStorageCommand.prepareDefaultMockBuckets()
})
it('Should succeed to upload to object storage with the appropriate proxy config', async function () {
this.timeout(120000)
await servers[0].kill()
await servers[0].run(ObjectStorageCommand.getDefaultConfig(), { env: goodEnv })
await servers[0].run(ObjectStorageCommand.getDefaultMockConfig(), { env: goodEnv })
const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
await waitJobs(servers)
const video = await servers[0].videos.get({ id: uuid })
expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
})
it('Should fail to upload to object storage with a wrong proxy config', async function () {
this.timeout(120000)
await servers[0].kill()
await servers[0].run(ObjectStorageCommand.getDefaultConfig(), { env: badEnv })
await servers[0].run(ObjectStorageCommand.getDefaultMockConfig(), { env: badEnv })
const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
await waitJobs(servers)
const video = await servers[0].videos.get({ id: uuid })
expectNotStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
expectNotStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
})
})

View File

@ -2,7 +2,7 @@
import { expect } from 'chai'
import { checkResolutionsInMasterPlaylist, expectStartWith } from '@server/tests/shared'
import { areObjectStorageTestsDisabled } from '@shared/core-utils'
import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
import { HttpStatusCode, VideoDetails } from '@shared/models'
import {
cleanupTests,
@ -19,7 +19,7 @@ import {
async function checkFilesInObjectStorage (video: VideoDetails) {
for (const file of video.files) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
expectStartWith(file.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
@ -27,14 +27,14 @@ async function checkFilesInObjectStorage (video: VideoDetails) {
const hlsPlaylist = video.streamingPlaylists[0]
for (const file of hlsPlaylist.files) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
expectStartWith(hlsPlaylist.playlistUrl, ObjectStorageCommand.getPlaylistBaseUrl())
expectStartWith(hlsPlaylist.playlistUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
await makeRawRequest({ url: hlsPlaylist.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
expectStartWith(hlsPlaylist.segmentsSha256Url, ObjectStorageCommand.getPlaylistBaseUrl())
expectStartWith(hlsPlaylist.segmentsSha256Url, ObjectStorageCommand.getMockPlaylistBaseUrl())
await makeRawRequest({ url: hlsPlaylist.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
}
@ -49,7 +49,7 @@ function runTests (objectStorage: boolean) {
this.timeout(120000)
const config = objectStorage
? ObjectStorageCommand.getDefaultConfig()
? ObjectStorageCommand.getDefaultMockConfig()
: {}
// Run server 2 to have transcoding enabled
@ -60,7 +60,7 @@ function runTests (objectStorage: boolean) {
await doubleFollow(servers[0], servers[1])
if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets()
if (objectStorage) await ObjectStorageCommand.prepareDefaultMockBuckets()
const { shortUUID } = await servers[0].videos.quickUpload({ name: 'video' })
videoUUID = shortUUID
@ -256,7 +256,7 @@ describe('Test create transcoding jobs from API', function () {
})
describe('On object storage', function () {
if (areObjectStorageTestsDisabled()) return
if (areMockObjectStorageTestsDisabled()) return
runTests(true)
})

View File

@ -2,7 +2,7 @@
import { join } from 'path'
import { checkDirectoryIsEmpty, checkTmpIsEmpty, completeCheckHlsPlaylist } from '@server/tests/shared'
import { areObjectStorageTestsDisabled } from '@shared/core-utils'
import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
import { HttpStatusCode } from '@shared/models'
import {
cleanupTests,
@ -150,19 +150,19 @@ describe('Test HLS videos', function () {
})
describe('With object storage enabled', function () {
if (areObjectStorageTestsDisabled()) return
if (areMockObjectStorageTestsDisabled()) return
before(async function () {
this.timeout(120000)
const configOverride = ObjectStorageCommand.getDefaultConfig()
await ObjectStorageCommand.prepareDefaultBuckets()
const configOverride = ObjectStorageCommand.getDefaultMockConfig()
await ObjectStorageCommand.prepareDefaultMockBuckets()
await servers[0].kill()
await servers[0].run(configOverride)
})
runTestSuite(true, ObjectStorageCommand.getPlaylistBaseUrl())
runTestSuite(true, ObjectStorageCommand.getMockPlaylistBaseUrl())
})
after(async function () {

View File

@ -1,7 +1,7 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { completeCheckHlsPlaylist } from '@server/tests/shared'
import { areObjectStorageTestsDisabled, wait } from '@shared/core-utils'
import { areMockObjectStorageTestsDisabled, wait } from '@shared/core-utils'
import { VideoPrivacy } from '@shared/models'
import {
cleanupTests,
@ -130,19 +130,19 @@ describe('Test update video privacy while transcoding', function () {
})
describe('With object storage enabled', function () {
if (areObjectStorageTestsDisabled()) return
if (areMockObjectStorageTestsDisabled()) return
before(async function () {
this.timeout(120000)
const configOverride = ObjectStorageCommand.getDefaultConfig()
await ObjectStorageCommand.prepareDefaultBuckets()
const configOverride = ObjectStorageCommand.getDefaultMockConfig()
await ObjectStorageCommand.prepareDefaultMockBuckets()
await servers[0].kill()
await servers[0].run(configOverride)
})
runTestSuite(true, ObjectStorageCommand.getPlaylistBaseUrl())
runTestSuite(true, ObjectStorageCommand.getMockPlaylistBaseUrl())
})
after(async function () {

View File

@ -1,6 +1,6 @@
import { expect } from 'chai'
import { expectStartWith } from '@server/tests/shared'
import { areObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils'
import { areMockObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils'
import { VideoStudioTask } from '@shared/models'
import {
cleanupTests,
@ -315,13 +315,13 @@ describe('Test video studio', function () {
})
describe('Object storage video edition', function () {
if (areObjectStorageTestsDisabled()) return
if (areMockObjectStorageTestsDisabled()) return
before(async function () {
await ObjectStorageCommand.prepareDefaultBuckets()
await ObjectStorageCommand.prepareDefaultMockBuckets()
await servers[0].kill()
await servers[0].run(ObjectStorageCommand.getDefaultConfig())
await servers[0].run(ObjectStorageCommand.getDefaultMockConfig())
await servers[0].config.enableMinimumTranscoding()
})
@ -344,11 +344,11 @@ describe('Test video studio', function () {
}
for (const webtorrentFile of video.files) {
expectStartWith(webtorrentFile.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
expectStartWith(webtorrentFile.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
}
for (const hlsFile of video.streamingPlaylists[0].files) {
expectStartWith(hlsFile.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
expectStartWith(hlsFile.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
}
await checkDuration(server, 9)

View File

@ -37,7 +37,7 @@ describe('Test video static file privacy', function () {
function runSuite () {
async function checkPrivateWebTorrentFiles (uuid: string) {
async function checkPrivateFiles (uuid: string) {
const video = await server.videos.getWithToken({ id: uuid })
for (const file of video.files) {
@ -63,7 +63,7 @@ describe('Test video static file privacy', function () {
}
}
async function checkPublicWebTorrentFiles (uuid: string) {
async function checkPublicFiles (uuid: string) {
const video = await server.videos.get({ id: uuid })
for (const file of getAllFiles(video)) {
@ -98,7 +98,7 @@ describe('Test video static file privacy', function () {
const { uuid } = await server.videos.quickUpload({ name: 'video', privacy })
await waitJobs([ server ])
await checkPrivateWebTorrentFiles(uuid)
await checkPrivateFiles(uuid)
}
})
@ -112,7 +112,7 @@ describe('Test video static file privacy', function () {
await server.videos.update({ id: uuid, attributes: { privacy } })
await waitJobs([ server ])
await checkPrivateWebTorrentFiles(uuid)
await checkPrivateFiles(uuid)
}
})
@ -125,7 +125,7 @@ describe('Test video static file privacy', function () {
await server.videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.UNLISTED } })
await waitJobs([ server ])
await checkPublicWebTorrentFiles(uuid)
await checkPublicFiles(uuid)
})
it('Should upload an internal video and update it to public to have a public static path', async function () {
@ -137,7 +137,7 @@ describe('Test video static file privacy', function () {
await server.videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } })
await waitJobs([ server ])
await checkPublicWebTorrentFiles(uuid)
await checkPublicFiles(uuid)
})
it('Should upload an internal video and schedule a public publish', async function () {
@ -160,7 +160,7 @@ describe('Test video static file privacy', function () {
await waitJobs([ server ])
await checkPublicWebTorrentFiles(uuid)
await checkPublicFiles(uuid)
})
}

View File

@ -1,7 +1,7 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { expect } from 'chai'
import { areObjectStorageTestsDisabled } from '@shared/core-utils'
import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
import { HttpStatusCode, VideoDetails, VideoFile, VideoInclude } from '@shared/models'
import {
cleanupTests,
@ -27,7 +27,7 @@ function assertVideoProperties (video: VideoFile, resolution: number, extname: s
async function checkFiles (video: VideoDetails, objectStorage: boolean) {
for (const file of video.files) {
if (objectStorage) expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
if (objectStorage) expectStartWith(file.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
@ -43,7 +43,7 @@ function runTests (objectStorage: boolean) {
this.timeout(90000)
const config = objectStorage
? ObjectStorageCommand.getDefaultConfig()
? ObjectStorageCommand.getDefaultMockConfig()
: {}
// Run server 2 to have transcoding enabled
@ -52,7 +52,7 @@ function runTests (objectStorage: boolean) {
await doubleFollow(servers[0], servers[1])
if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets()
if (objectStorage) await ObjectStorageCommand.prepareDefaultMockBuckets()
// Upload two videos for our needs
{
@ -157,7 +157,7 @@ describe('Test create import video jobs', function () {
})
describe('On object storage', function () {
if (areObjectStorageTestsDisabled()) return
if (areMockObjectStorageTestsDisabled()) return
runTests(true)
})

View File

@ -1,6 +1,6 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { areObjectStorageTestsDisabled } from '@shared/core-utils'
import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
import { HttpStatusCode, VideoDetails } from '@shared/models'
import {
cleanupTests,
@ -17,7 +17,7 @@ import { expectStartWith } from '../shared'
async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObjectStorage: boolean) {
for (const file of video.files) {
const start = inObjectStorage
? ObjectStorageCommand.getWebTorrentBaseUrl()
? ObjectStorageCommand.getMockWebTorrentBaseUrl()
: origin.url
expectStartWith(file.fileUrl, start)
@ -26,7 +26,7 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObject
}
const start = inObjectStorage
? ObjectStorageCommand.getPlaylistBaseUrl()
? ObjectStorageCommand.getMockPlaylistBaseUrl()
: origin.url
const hls = video.streamingPlaylists[0]
@ -41,7 +41,7 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObject
}
describe('Test create move video storage job', function () {
if (areObjectStorageTestsDisabled()) return
if (areMockObjectStorageTestsDisabled()) return
let servers: PeerTubeServer[] = []
const uuids: string[] = []
@ -55,7 +55,7 @@ describe('Test create move video storage job', function () {
await doubleFollow(servers[0], servers[1])
await ObjectStorageCommand.prepareDefaultBuckets()
await ObjectStorageCommand.prepareDefaultMockBuckets()
await servers[0].config.enableTranscoding()
@ -67,14 +67,14 @@ describe('Test create move video storage job', function () {
await waitJobs(servers)
await servers[0].kill()
await servers[0].run(ObjectStorageCommand.getDefaultConfig())
await servers[0].run(ObjectStorageCommand.getDefaultMockConfig())
})
it('Should move only one file', async function () {
this.timeout(120000)
const command = `npm run create-move-video-storage-job -- --to-object-storage -v ${uuids[1]}`
await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultConfig())
await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultMockConfig())
await waitJobs(servers)
for (const server of servers) {
@ -94,7 +94,7 @@ describe('Test create move video storage job', function () {
this.timeout(120000)
const command = `npm run create-move-video-storage-job -- --to-object-storage --all-videos`
await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultConfig())
await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultMockConfig())
await waitJobs(servers)
for (const server of servers) {

View File

@ -1,7 +1,7 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { expect } from 'chai'
import { areObjectStorageTestsDisabled } from '@shared/core-utils'
import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
import { HttpStatusCode, VideoFile } from '@shared/models'
import {
cleanupTests,
@ -18,8 +18,8 @@ import { checkResolutionsInMasterPlaylist, expectStartWith } from '../shared'
async function checkFilesInObjectStorage (files: VideoFile[], type: 'webtorrent' | 'playlist') {
for (const file of files) {
const shouldStartWith = type === 'webtorrent'
? ObjectStorageCommand.getWebTorrentBaseUrl()
: ObjectStorageCommand.getPlaylistBaseUrl()
? ObjectStorageCommand.getMockWebTorrentBaseUrl()
: ObjectStorageCommand.getMockPlaylistBaseUrl()
expectStartWith(file.fileUrl, shouldStartWith)
@ -36,7 +36,7 @@ function runTests (objectStorage: boolean) {
this.timeout(120000)
const config = objectStorage
? ObjectStorageCommand.getDefaultConfig()
? ObjectStorageCommand.getDefaultMockConfig()
: {}
// Run server 2 to have transcoding enabled
@ -47,7 +47,7 @@ function runTests (objectStorage: boolean) {
await doubleFollow(servers[0], servers[1])
if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets()
if (objectStorage) await ObjectStorageCommand.prepareDefaultMockBuckets()
for (let i = 1; i <= 5; i++) {
const { uuid, shortUUID } = await servers[0].videos.upload({ attributes: { name: 'video' + i } })
@ -255,7 +255,7 @@ describe('Test create transcoding jobs', function () {
})
describe('On object storage', function () {
if (areObjectStorageTestsDisabled()) return
if (areMockObjectStorageTestsDisabled()) return
runTests(true)
})

View File

@ -50,7 +50,7 @@ async function testVideoResolutions (options: {
})
if (objectStorage) {
expect(hlsPlaylist.playlistUrl).to.contain(ObjectStorageCommand.getPlaylistBaseUrl())
expect(hlsPlaylist.playlistUrl).to.contain(ObjectStorageCommand.getMockPlaylistBaseUrl())
}
for (let i = 0; i < resolutions.length; i++) {
@ -65,11 +65,11 @@ async function testVideoResolutions (options: {
})
const baseUrl = objectStorage
? ObjectStorageCommand.getPlaylistBaseUrl() + 'hls'
? ObjectStorageCommand.getMockPlaylistBaseUrl() + 'hls'
: originServer.url + '/static/streaming-playlists/hls'
if (objectStorage) {
expect(hlsPlaylist.segmentsSha256Url).to.contain(ObjectStorageCommand.getPlaylistBaseUrl())
expect(hlsPlaylist.segmentsSha256Url).to.contain(ObjectStorageCommand.getMockPlaylistBaseUrl())
}
const subPlaylist = await originServer.streamingPlaylists.get({

View File

@ -12,7 +12,7 @@ export class MockObjectStorage {
const app = express()
app.get('/:bucketName/:path(*)', (req: express.Request, res: express.Response, next: express.NextFunction) => {
const url = `http://${req.params.bucketName}.${ObjectStorageCommand.getEndpointHost()}/${req.params.path}`
const url = `http://${req.params.bucketName}.${ObjectStorageCommand.getMockEndpointHost()}/${req.params.path}`
if (process.env.DEBUG) {
console.log('Receiving request on mocked server %s.', req.url)

View File

@ -97,7 +97,7 @@ declare module 'express' {
title?: string
status?: number
type?: ServerErrorCode
type?: ServerErrorCode | string
instance?: string
data?: PeerTubeProblemDocumentData

View File

@ -14,7 +14,7 @@ function areHttpImportTestsDisabled () {
return disabled
}
function areObjectStorageTestsDisabled () {
function areMockObjectStorageTestsDisabled () {
const disabled = process.env.ENABLE_OBJECT_STORAGE_TESTS !== 'true'
if (disabled) console.log('ENABLE_OBJECT_STORAGE_TESTS env is not set to "true" so object storage tests are disabled')
@ -22,9 +22,25 @@ function areObjectStorageTestsDisabled () {
return disabled
}
function areScalewayObjectStorageTestsDisabled () {
if (areMockObjectStorageTestsDisabled()) return true
const enabled = process.env.OBJECT_STORAGE_SCALEWAY_KEY_ID && process.env.OBJECT_STORAGE_SCALEWAY_ACCESS_KEY
if (!enabled) {
console.log(
'OBJECT_STORAGE_SCALEWAY_KEY_ID and/or OBJECT_STORAGE_SCALEWAY_ACCESS_KEY are not set, so scaleway object storage tests are disabled'
)
return true
}
return false
}
export {
parallelTests,
isGithubCI,
areHttpImportTestsDisabled,
areObjectStorageTestsDisabled
areMockObjectStorageTestsDisabled,
areScalewayObjectStorageTestsDisabled
}

View File

@ -1,5 +1,6 @@
import { VideoDetails } from '../../models/videos/video.model'
import { VideoStreamingPlaylistType } from '@shared/models'
import { VideoPrivacy } from '../../models/videos/video-privacy.enum'
import { VideoDetails } from '../../models/videos/video.model'
function getAllPrivacies () {
return [ VideoPrivacy.PUBLIC, VideoPrivacy.INTERNAL, VideoPrivacy.PRIVATE, VideoPrivacy.UNLISTED ]
@ -8,14 +9,18 @@ function getAllPrivacies () {
function getAllFiles (video: Partial<Pick<VideoDetails, 'files' | 'streamingPlaylists'>>) {
const files = video.files
if (video.streamingPlaylists[0]) {
return files.concat(video.streamingPlaylists[0].files)
}
const hls = getHLS(video)
if (hls) return files.concat(hls.files)
return files
}
function getHLS (video: Partial<Pick<VideoDetails, 'streamingPlaylists'>>) {
return video.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
}
export {
getAllPrivacies,
getAllFiles
getAllFiles,
getHLS
}

View File

@ -1,2 +1,2 @@
export * from './bitrate'
export * from './privacy'
export * from './common'

View File

@ -23,6 +23,11 @@ export class SQLCommand extends AbstractCommand {
return parseInt(total, 10)
}
async getInternalFileUrl (fileId: number) {
return this.selectQuery(`SELECT "fileUrl" FROM "videoFile" WHERE id = ${fileId}`)
.then(rows => rows[0].fileUrl as string)
}
setActorField (to: string, field: string, value: string) {
const seq = this.getSequelize()

View File

@ -4,74 +4,121 @@ import { makePostBodyRequest } from '../requests'
import { AbstractCommand } from '../shared'
export class ObjectStorageCommand extends AbstractCommand {
static readonly DEFAULT_PLAYLIST_BUCKET = 'streaming-playlists'
static readonly DEFAULT_WEBTORRENT_BUCKET = 'videos'
static readonly DEFAULT_PLAYLIST_MOCK_BUCKET = 'streaming-playlists'
static readonly DEFAULT_WEBTORRENT_MOCK_BUCKET = 'videos'
static getDefaultConfig () {
static readonly DEFAULT_SCALEWAY_BUCKET = 'peertube-ci-test'
// ---------------------------------------------------------------------------
static getDefaultMockConfig () {
return {
object_storage: {
enabled: true,
endpoint: 'http://' + this.getEndpointHost(),
region: this.getRegion(),
endpoint: 'http://' + this.getMockEndpointHost(),
region: this.getMockRegion(),
credentials: this.getCredentialsConfig(),
credentials: this.getMockCredentialsConfig(),
streaming_playlists: {
bucket_name: this.DEFAULT_PLAYLIST_BUCKET
bucket_name: this.DEFAULT_PLAYLIST_MOCK_BUCKET
},
videos: {
bucket_name: this.DEFAULT_WEBTORRENT_BUCKET
bucket_name: this.DEFAULT_WEBTORRENT_MOCK_BUCKET
}
}
}
}
static getCredentialsConfig () {
static getMockCredentialsConfig () {
return {
access_key_id: 'AKIAIOSFODNN7EXAMPLE',
secret_access_key: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
}
}
static getEndpointHost () {
static getMockEndpointHost () {
return 'localhost:9444'
}
static getRegion () {
static getMockRegion () {
return 'us-east-1'
}
static getWebTorrentBaseUrl () {
return `http://${this.DEFAULT_WEBTORRENT_BUCKET}.${this.getEndpointHost()}/`
static getMockWebTorrentBaseUrl () {
return `http://${this.DEFAULT_WEBTORRENT_MOCK_BUCKET}.${this.getMockEndpointHost()}/`
}
static getPlaylistBaseUrl () {
return `http://${this.DEFAULT_PLAYLIST_BUCKET}.${this.getEndpointHost()}/`
static getMockPlaylistBaseUrl () {
return `http://${this.DEFAULT_PLAYLIST_MOCK_BUCKET}.${this.getMockEndpointHost()}/`
}
static async prepareDefaultBuckets () {
await this.createBucket(this.DEFAULT_PLAYLIST_BUCKET)
await this.createBucket(this.DEFAULT_WEBTORRENT_BUCKET)
static async prepareDefaultMockBuckets () {
await this.createMockBucket(this.DEFAULT_PLAYLIST_MOCK_BUCKET)
await this.createMockBucket(this.DEFAULT_WEBTORRENT_MOCK_BUCKET)
}
static async createBucket (name: string) {
static async createMockBucket (name: string) {
await makePostBodyRequest({
url: this.getEndpointHost(),
url: this.getMockEndpointHost(),
path: '/ui/' + name + '?delete',
expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307
})
await makePostBodyRequest({
url: this.getEndpointHost(),
url: this.getMockEndpointHost(),
path: '/ui/' + name + '?create',
expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307
})
await makePostBodyRequest({
url: this.getEndpointHost(),
url: this.getMockEndpointHost(),
path: '/ui/' + name + '?make-public',
expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307
})
}
// ---------------------------------------------------------------------------
static getDefaultScalewayConfig (serverNumber: number) {
return {
object_storage: {
enabled: true,
endpoint: this.getScalewayEndpointHost(),
region: this.getScalewayRegion(),
credentials: this.getScalewayCredentialsConfig(),
streaming_playlists: {
bucket_name: this.DEFAULT_SCALEWAY_BUCKET,
prefix: `test:server-${serverNumber}-streaming-playlists:`
},
videos: {
bucket_name: this.DEFAULT_SCALEWAY_BUCKET,
prefix: `test:server-${serverNumber}-videos:`
}
}
}
}
static getScalewayCredentialsConfig () {
return {
access_key_id: process.env.OBJECT_STORAGE_SCALEWAY_KEY_ID,
secret_access_key: process.env.OBJECT_STORAGE_SCALEWAY_ACCESS_KEY
}
}
static getScalewayEndpointHost () {
return 's3.fr-par.scw.cloud'
}
static getScalewayRegion () {
return 'fr-par'
}
static getScalewayBaseUrl () {
return `https://${this.DEFAULT_SCALEWAY_BUCKET}.${this.getScalewayEndpointHost()}/`
}
}

View File

@ -197,7 +197,7 @@ export class LiveCommand extends AbstractCommand {
const segmentName = `${playlistNumber}-00000${segment}.ts`
const baseUrl = objectStorage
? ObjectStorageCommand.getPlaylistBaseUrl() + 'hls'
? ObjectStorageCommand.getMockPlaylistBaseUrl() + 'hls'
: server.url + '/static/streaming-playlists/hls'
let error = true
@ -253,7 +253,7 @@ export class LiveCommand extends AbstractCommand {
const segmentName = `${playlistNumber}-00000${segment}.ts`
const baseUrl = objectStorage
? ObjectStorageCommand.getPlaylistBaseUrl()
? ObjectStorageCommand.getMockPlaylistBaseUrl()
: `${this.server.url}/static/streaming-playlists/hls`
const url = `${baseUrl}/${videoUUID}/${segmentName}`
@ -275,7 +275,7 @@ export class LiveCommand extends AbstractCommand {
const { playlistName, videoUUID, objectStorage = false } = options
const baseUrl = objectStorage
? ObjectStorageCommand.getPlaylistBaseUrl()
? ObjectStorageCommand.getMockPlaylistBaseUrl()
: `${this.server.url}/static/streaming-playlists/hls`
const url = `${baseUrl}/${videoUUID}/${playlistName}`

1151
yarn.lock

File diff suppressed because it is too large Load Diff