From 7c9f07e14065f2d12cefedf43e05a9ceb6acaa1d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 31 May 2024 11:31:52 +0200 Subject: [PATCH] Improve `Video` AP compatibility Compat with text/html descriptions Compat with SPDX for licences Compat with missing sensitive attribute Compat with missing tag attribute Compat with missing video file magnet URI Compat with missing streaming playlist segmentsSha256Url Compat with optional comments/likes/dislikes/shares URI in video object Add more debug logs when the object is not valid --- .../metadata/video-description.component.ts | 2 +- .../video-download.component.html | 2 +- .../p2p-media-loader-plugin.ts | 10 +- .../p2p-media-loader/segment-url-builder.ts | 10 +- .../hls-options-builder.ts | 43 ++++-- .../player/types/peertube-videojs-typings.ts | 5 +- .../custom-validators/activitypub/videos.ts | 128 +++++++++--------- server/core/helpers/video.ts | 24 ++++ server/core/initializers/constants.ts | 2 +- .../0850-streaming-playlist-sha-nullable.ts | 25 ++++ .../shared/object-to-model-attributes.ts | 80 +++++++---- .../videos/shared/video-sync-attributes.ts | 4 + .../models/video/video-streaming-playlist.ts | 4 +- 13 files changed, 215 insertions(+), 124 deletions(-) create mode 100644 server/core/initializers/migrations/0850-streaming-playlist-sha-nullable.ts diff --git a/client/src/app/+videos/+video-watch/shared/metadata/video-description.component.ts b/client/src/app/+videos/+video-watch/shared/metadata/video-description.component.ts index aa5940888..cc8cb5ee2 100644 --- a/client/src/app/+videos/+video-watch/shared/metadata/video-description.component.ts +++ b/client/src/app/+videos/+video-watch/shared/metadata/video-description.component.ts @@ -52,7 +52,7 @@ export class VideoDescriptionComponent implements OnChanges { } private async setVideoDescriptionHTML () { - const html = await this.markdownService.textMarkdownToHTML({ markdown: this.video.description }) + const html = await this.markdownService.textMarkdownToHTML({ markdown: this.video.description, withHtml: true, withEmoji: true }) this.videoHTMLDescription = this.markdownService.processVideoTimestamps(this.video.shortUUID, html) } diff --git a/client/src/app/shared/shared-video-miniature/video-download.component.html b/client/src/app/shared/shared-video-miniature/video-download.component.html index dd76754bb..ced76fbe8 100644 --- a/client/src/app/shared/shared-video-miniature/video-download.component.html +++ b/client/src/app/shared/shared-video-miniature/video-download.component.html @@ -129,7 +129,7 @@
-
+
diff --git a/client/src/assets/player/shared/p2p-media-loader/p2p-media-loader-plugin.ts b/client/src/assets/player/shared/p2p-media-loader/p2p-media-loader-plugin.ts index a05c1a3e2..183e14d93 100644 --- a/client/src/assets/player/shared/p2p-media-loader/p2p-media-loader-plugin.ts +++ b/client/src/assets/player/shared/p2p-media-loader/p2p-media-loader-plugin.ts @@ -121,10 +121,16 @@ class P2pMediaLoaderPlugin extends Plugin { logger.error(`Segment ${segment.id} error.`, err) - this.options.redundancyUrlManager.removeBySegmentUrl(segment.requestUrl) + if (this.options.redundancyUrlManager) { + this.options.redundancyUrlManager.removeBySegmentUrl(segment.requestUrl) + } }) - this.statsP2PBytes.peersWithWebSeed = 1 + this.options.redundancyUrlManager.countBaseUrls() + const redundancyUrlsCount = this.options.redundancyUrlManager + ? this.options.redundancyUrlManager.countBaseUrls() + : 0 + + this.statsP2PBytes.peersWithWebSeed = 1 + redundancyUrlsCount this.runStats() diff --git a/client/src/assets/player/shared/p2p-media-loader/segment-url-builder.ts b/client/src/assets/player/shared/p2p-media-loader/segment-url-builder.ts index ad0e460ae..8875e5353 100644 --- a/client/src/assets/player/shared/p2p-media-loader/segment-url-builder.ts +++ b/client/src/assets/player/shared/p2p-media-loader/segment-url-builder.ts @@ -1,14 +1,10 @@ import { Segment } from '@peertube/p2p-media-loader-core' import { RedundancyUrlManager } from './redundancy-url-manager' -function segmentUrlBuilderFactory (redundancyUrlManager: RedundancyUrlManager) { +export function segmentUrlBuilderFactory (redundancyUrlManager: RedundancyUrlManager | null) { return function segmentBuilder (segment: Segment) { + if (!redundancyUrlManager) return segment.url + return redundancyUrlManager.buildUrl(segment.url) } } - -// --------------------------------------------------------------------------- - -export { - segmentUrlBuilderFactory -} diff --git a/client/src/assets/player/shared/player-options-builder/hls-options-builder.ts b/client/src/assets/player/shared/player-options-builder/hls-options-builder.ts index d190d99c0..b7733471b 100644 --- a/client/src/assets/player/shared/player-options-builder/hls-options-builder.ts +++ b/client/src/assets/player/shared/player-options-builder/hls-options-builder.ts @@ -1,14 +1,14 @@ import { HybridLoaderSettings } from '@peertube/p2p-media-loader-core' import { HlsJsEngineSettings } from '@peertube/p2p-media-loader-hlsjs' -import { logger } from '@root-helpers/logger' import { LiveVideoLatencyMode } from '@peertube/peertube-models' +import { logger } from '@root-helpers/logger' +import { peertubeLocalStorage } from '@root-helpers/peertube-web-storage' import { getAverageBandwidthInStore } from '../../peertube-player-local-storage' import { P2PMediaLoader, P2PMediaLoaderPluginOptions, PeerTubePlayerContructorOptions, PeerTubePlayerLoadOptions } from '../../types' import { getRtcConfig, isSameOrigin } from '../common' import { RedundancyUrlManager } from '../p2p-media-loader/redundancy-url-manager' import { segmentUrlBuilderFactory } from '../p2p-media-loader/segment-url-builder' import { SegmentValidator } from '../p2p-media-loader/segment-validator' -import { peertubeLocalStorage } from '@root-helpers/peertube-web-storage' type ConstructorOptions = Pick & @@ -25,15 +25,26 @@ export class HLSOptionsBuilder { } async getPluginOptions () { - const redundancyUrlManager = new RedundancyUrlManager(this.options.hls.redundancyBaseUrls) - const segmentValidator = new SegmentValidator({ - segmentsSha256Url: this.options.hls.segmentsSha256Url, - authorizationHeader: this.options.authorizationHeader, - requiresUserAuth: this.options.requiresUserAuth, - serverUrl: this.options.serverUrl, - requiresPassword: this.options.requiresPassword, - videoPassword: this.options.videoPassword - }) + const segmentsSha256Url = this.options.hls.segmentsSha256Url + + if (!segmentsSha256Url) { + logger.info('No segmentsSha256Url found. Disabling P2P & redundancy.') + } + + const redundancyUrlManager = segmentsSha256Url + ? new RedundancyUrlManager(this.options.hls.redundancyBaseUrls) + : null + + const segmentValidator = segmentsSha256Url + ? new SegmentValidator({ + segmentsSha256Url, + authorizationHeader: this.options.authorizationHeader, + requiresUserAuth: this.options.requiresUserAuth, + serverUrl: this.options.serverUrl, + requiresPassword: this.options.requiresPassword, + videoPassword: this.options.videoPassword + }) + : null const p2pMediaLoaderConfig = await this.options.pluginsManager.runHook( 'filter:internal.player.p2p-media-loader.options.result', @@ -45,7 +56,7 @@ export class HLSOptionsBuilder { requiresUserAuth: this.options.requiresUserAuth, videoFileToken: this.options.videoFileToken, - p2pEnabled: this.options.p2pEnabled, + p2pEnabled: segmentsSha256Url && this.options.p2pEnabled, redundancyUrlManager, type: 'application/x-mpegURL', @@ -77,8 +88,8 @@ export class HLSOptionsBuilder { // --------------------------------------------------------------------------- private getP2PMediaLoaderOptions (options: { - redundancyUrlManager: RedundancyUrlManager - segmentValidator: SegmentValidator + redundancyUrlManager: RedundancyUrlManager | null + segmentValidator: SegmentValidator | null }): HlsJsEngineSettings { const { redundancyUrlManager, segmentValidator } = options @@ -117,7 +128,9 @@ export class HLSOptionsBuilder { else xhr.setRequestHeader('Authorization', this.options.authorizationHeader()) }, - segmentValidator: segmentValidator.validate.bind(segmentValidator), + segmentValidator: segmentValidator + ? segmentValidator.validate.bind(segmentValidator) + : null, segmentUrlBuilder: segmentUrlBuilderFactory(redundancyUrlManager), diff --git a/client/src/assets/player/types/peertube-videojs-typings.ts b/client/src/assets/player/types/peertube-videojs-typings.ts index 7aec818ad..692009fe2 100644 --- a/client/src/assets/player/types/peertube-videojs-typings.ts +++ b/client/src/assets/player/types/peertube-videojs-typings.ts @@ -198,14 +198,15 @@ type WebVideoPluginOptions = { } type P2PMediaLoaderPluginOptions = { - redundancyUrlManager: RedundancyUrlManager + redundancyUrlManager: RedundancyUrlManager | null + segmentValidator: SegmentValidator | null + type: string src: string p2pEnabled: boolean loader: P2PMediaLoader - segmentValidator: SegmentValidator requiresUserAuth: boolean videoFileToken: () => string diff --git a/server/core/helpers/custom-validators/activitypub/videos.ts b/server/core/helpers/custom-validators/activitypub/videos.ts index 255d051d7..51f66134e 100644 --- a/server/core/helpers/custom-validators/activitypub/videos.ts +++ b/server/core/helpers/custom-validators/activitypub/videos.ts @@ -8,10 +8,11 @@ import { VideoState } from '@peertube/peertube-models' import { logger } from '@server/helpers/logger.js' +import { spdxToPeertubeLicence } from '@server/helpers/video.js' import validator from 'validator' import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../../initializers/constants.js' import { peertubeTruncate } from '../../core-utils.js' -import { isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc.js' +import { exists, isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc.js' import { isLiveLatencyModeValid } from '../video-lives.js' import { isVideoCommentsPolicyValid, @@ -24,43 +25,28 @@ import { } from '../videos.js' import { isActivityPubUrlValid, isActivityPubVideoDurationValid, isBaseActivityValid, setValidAttributedTo } from './misc.js' -function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) { +export function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) { return isBaseActivityValid(activity, 'Update') && sanitizeAndCheckVideoTorrentObject(activity.object) } -function sanitizeAndCheckVideoTorrentObject (video: VideoObject) { +export function sanitizeAndCheckVideoTorrentObject (video: VideoObject) { if (!video || video.type !== 'Video') return false - if (!setValidRemoteTags(video)) { - logger.debug('Video has invalid tags', { video }) - return false - } - if (!setValidRemoteVideoUrls(video)) { - logger.debug('Video has invalid urls', { video }) - return false - } - if (!setRemoteVideoContent(video)) { - logger.debug('Video has invalid content', { video }) - return false - } - if (!setValidAttributedTo(video)) { - logger.debug('Video has invalid attributedTo', { video }) - return false - } - if (!setValidRemoteCaptions(video)) { - logger.debug('Video has invalid captions', { video }) - return false - } - if (!setValidRemoteIcon(video)) { - logger.debug('Video has invalid icons', { video }) - return false - } - if (!setValidStoryboard(video)) { - logger.debug('Video has invalid preview (storyboard)', { video }) + const fail = (field: string) => { + logger.debug(`Video field is not valid to PeerTube: ${field}`, { video }) return false } + if (!setValidRemoteTags(video)) return fail('tags') + if (!setValidRemoteVideoUrls(video)) return fail('urls') + if (!setRemoteVideoContent(video)) return fail('content') + if (!setValidAttributedTo(video)) return fail('attributedTo') + if (!setValidRemoteCaptions(video)) return fail('captions') + if (!setValidRemoteIcon(video)) return fail('icons') + if (!setValidStoryboard(video)) return fail('preview (storyboard)') + if (!setValidLicence(video)) return fail('licence') + // TODO: compat with < 6.1, remove in 7.0 if (!video.uuid && video['identifier']) video.uuid = video['identifier'] @@ -71,6 +57,7 @@ function sanitizeAndCheckVideoTorrentObject (video: VideoObject) { if (!isBooleanValid(video.isLiveBroadcast)) video.isLiveBroadcast = false if (!isBooleanValid(video.liveSaveReplay)) video.liveSaveReplay = false if (!isBooleanValid(video.permanentLive)) video.permanentLive = false + if (!isBooleanValid(video.sensitive)) video.sensitive = false if (!isLiveLatencyModeValid(video.latencyMode)) video.latencyMode = LiveVideoLatencyMode.DEFAULT if (video.commentsPolicy) { @@ -83,25 +70,31 @@ function sanitizeAndCheckVideoTorrentObject (video: VideoObject) { video.commentsPolicy = VideoCommentPolicy.DISABLED } - return isActivityPubUrlValid(video.id) && - isVideoNameValid(video.name) && - isActivityPubVideoDurationValid(video.duration) && - isVideoDurationValid(video.duration.replace(/[^0-9]+/g, '')) && - isUUIDValid(video.uuid) && - (!video.category || isRemoteNumberIdentifierValid(video.category)) && - (!video.licence || isRemoteNumberIdentifierValid(video.licence)) && - (!video.language || isRemoteStringIdentifierValid(video.language)) && - isVideoViewsValid(video.views) && - isBooleanValid(video.sensitive) && - isDateValid(video.published) && - isDateValid(video.updated) && - (!video.originallyPublishedAt || isDateValid(video.originallyPublishedAt)) && - (!video.uploadDate || isDateValid(video.uploadDate)) && - (!video.content || isRemoteVideoContentValid(video.mediaType, video.content)) && - video.attributedTo.length !== 0 + if (!isActivityPubUrlValid(video.id)) return fail('id') + if (!isVideoNameValid(video.name)) return fail('name') + + if (!isActivityPubVideoDurationValid(video.duration)) return fail('duration format') + if (!isVideoDurationValid(video.duration.replace(/[^0-9]+/g, ''))) return fail('duration') + + if (!isUUIDValid(video.uuid)) return fail('uuid') + + if (exists(video.category) && !isRemoteNumberIdentifierValid(video.category)) return fail('category') + if (exists(video.language) && !isRemoteStringIdentifierValid(video.language)) return fail('language') + + if (!isVideoViewsValid(video.views)) return fail('views') + if (!isDateValid(video.published)) return fail('published') + if (!isDateValid(video.updated)) return fail('updated') + + if (exists(video.originallyPublishedAt) && !isDateValid(video.originallyPublishedAt)) return fail('originallyPublishedAt') + if (exists(video.uploadDate) && !isDateValid(video.uploadDate)) return fail('uploadDate') + if (exists(video.content) && !isRemoteVideoContentValid(video.mediaType, video.content)) return fail('mediaType/content') + + if (video.attributedTo.length === 0) return fail('attributedTo') + + return true } -function isRemoteVideoUrlValid (url: any) { +export function isRemoteVideoUrlValid (url: any) { return url.type === 'Link' && // Video file link ( @@ -133,44 +126,32 @@ function isRemoteVideoUrlValid (url: any) { isAPVideoFileUrlMetadataObject(url) } -function isAPVideoFileUrlMetadataObject (url: any): url is ActivityVideoFileMetadataUrlObject { +export function isAPVideoFileUrlMetadataObject (url: any): url is ActivityVideoFileMetadataUrlObject { return url && url.type === 'Link' && url.mediaType === 'application/json' && isArray(url.rel) && url.rel.includes('metadata') } -function isAPVideoTrackerUrlObject (url: any): url is ActivityTrackerUrlObject { +export function isAPVideoTrackerUrlObject (url: any): url is ActivityTrackerUrlObject { return isArray(url.rel) && url.rel.includes('tracker') && isActivityPubUrlValid(url.href) } // --------------------------------------------------------------------------- - -export { - isAPVideoFileUrlMetadataObject, - isAPVideoTrackerUrlObject, - isRemoteStringIdentifierValid, - isRemoteVideoUrlValid, - sanitizeAndCheckVideoTorrentObject, - sanitizeAndCheckVideoTorrentUpdateActivity -} - +// Private // --------------------------------------------------------------------------- -function setValidRemoteTags (video: any) { - if (Array.isArray(video.tag) === false) return false +function setValidRemoteTags (video: VideoObject) { + if (Array.isArray(video.tag) === false) video.tag = [] - video.tag = video.tag.filter(t => { - return t.type === 'Hashtag' && - isVideoTagValid(t.name) - }) + video.tag = video.tag.filter(t => t.type === 'Hashtag' && isVideoTagValid(t.name)) return true } -function setValidRemoteCaptions (video: any) { +function setValidRemoteCaptions (video: VideoObject) { if (!video.subtitleLanguage) video.subtitleLanguage = [] if (Array.isArray(video.subtitleLanguage) === false) return false @@ -193,7 +174,7 @@ function isRemoteStringIdentifierValid (data: any) { } function isRemoteVideoContentValid (mediaType: string, content: string) { - return mediaType === 'text/markdown' && isVideoDescriptionValid(content) + return (mediaType === 'text/markdown' || mediaType === 'text/html') && isVideoDescriptionValid(content) } function setValidRemoteIcon (video: any) { @@ -219,7 +200,7 @@ function setValidRemoteVideoUrls (video: any) { return true } -function setRemoteVideoContent (video: any) { +function setRemoteVideoContent (video: VideoObject) { if (video.content) { video.content = peertubeTruncate(video.content, { length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max }) } @@ -227,6 +208,19 @@ function setRemoteVideoContent (video: any) { return true } +function setValidLicence (video: VideoObject) { + if (!exists(video.licence)) return true + + if (validator.default.isInt(video.licence.identifier)) return isRemoteNumberIdentifierValid(video.licence) + + const spdx = spdxToPeertubeLicence(video.licence.identifier) + video.licence.identifier = spdx + ? spdx + '' + : undefined + + return true +} + function setValidStoryboard (video: VideoObject) { if (!video.preview) return true if (!Array.isArray(video.preview)) return false diff --git a/server/core/helpers/video.ts b/server/core/helpers/video.ts index 36e77f140..ff7539bc2 100644 --- a/server/core/helpers/video.ts +++ b/server/core/helpers/video.ts @@ -26,3 +26,27 @@ export function getExtFromMimetype (mimeTypes: { [id: string]: string | string[] return value } + +export function peertubeLicenceToSPDX (licence: number) { + return { + 1: 'CC-BY-4.0', + 2: 'CC-BY-SA-4.0', + 3: 'CC-BY-ND-4.0', + 4: 'CC-BY-NC-4.0', + 5: 'CC-BY-NC-SA-4.0', + 6: 'CC-BY-NC-ND-4.0', + 7: 'CC0' + }[licence] +} + +export function spdxToPeertubeLicence (licence: string) { + return { + 'CC-BY-4.0': 1, + 'CC-BY-SA-4.0': 2, + 'CC-BY-ND-4.0': 3, + 'CC-BY-NC-4.0': 4, + 'CC-BY-NC-SA-4.0': 5, + 'CC-BY-NC-ND-4.0': 6, + 'CC0': 7 + }[licence] +} diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.ts index 002ab9ee6..dde4c41e4 100644 --- a/server/core/initializers/constants.ts +++ b/server/core/initializers/constants.ts @@ -47,7 +47,7 @@ import { cpus } from 'os' // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 845 +const LAST_MIGRATION_VERSION = 850 // --------------------------------------------------------------------------- diff --git a/server/core/initializers/migrations/0850-streaming-playlist-sha-nullable.ts b/server/core/initializers/migrations/0850-streaming-playlist-sha-nullable.ts new file mode 100644 index 000000000..ce064e935 --- /dev/null +++ b/server/core/initializers/migrations/0850-streaming-playlist-sha-nullable.ts @@ -0,0 +1,25 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize +}): Promise { + const { transaction } = utils + + { + await utils.queryInterface.changeColumn('videoStreamingPlaylist', 'segmentsSha256Filename', { + type: Sequelize.STRING, + defaultValue: null, + allowNull: true + }, { transaction }) + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + down, up +} diff --git a/server/core/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/core/lib/activitypub/videos/shared/object-to-model-attributes.ts index 05aab3160..c87030019 100644 --- a/server/core/lib/activitypub/videos/shared/object-to-model-attributes.ts +++ b/server/core/lib/activitypub/videos/shared/object-to-model-attributes.ts @@ -16,7 +16,6 @@ import { isAPVideoFileUrlMetadataObject } from '@server/helpers/custom-validator import { isArray } from '@server/helpers/custom-validators/misc.js' import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/videos.js' import { generateImageFilename } from '@server/helpers/image-utils.js' -import { logger } from '@server/helpers/logger.js' import { getExtFromMimetype } from '@server/helpers/video.js' import { MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants.js' import { generateTorrentFileName } from '@server/lib/paths.js' @@ -58,21 +57,6 @@ export function getFileAttributesFromUrl ( const attributes: FilteredModelAttributes[] = [] for (const fileUrl of fileUrls) { - // Fetch associated magnet uri - const magnet = urls.filter(isAPMagnetUrlObject) - .find(u => u.height === fileUrl.height) - - if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href) - - const parsed = magnetUriDecode(magnet.href) - if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) { - throw new Error('Cannot parse magnet URI ' + magnet.href) - } - - const torrentUrl = Array.isArray(parsed.xs) - ? parsed.xs[0] - : parsed.xs - // Fetch associated metadata url, if any const metadata = urls.filter(isAPVideoFileUrlMetadataObject) .find(u => { @@ -84,14 +68,20 @@ export function getFileAttributesFromUrl ( const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType) const resolution = fileUrl.height const videoId = isStreamingPlaylist(videoOrPlaylist) ? null : videoOrPlaylist.id - const videoStreamingPlaylistId = isStreamingPlaylist(videoOrPlaylist) ? videoOrPlaylist.id : null + + const videoStreamingPlaylistId = isStreamingPlaylist(videoOrPlaylist) + ? videoOrPlaylist.id + : null + + const { torrentFilename, infoHash, torrentUrl } = getTorrentRelatedInfo({ videoOrPlaylist, urls, fileUrl }) const attribute = { extname, - infoHash: parsed.infoHash, resolution, + size: fileUrl.size, fps: fileUrl.fps || -1, + metadataUrl: metadata?.href, width: fileUrl.width, @@ -101,9 +91,9 @@ export function getFileAttributesFromUrl ( filename: basename(fileUrl.href), fileUrl: fileUrl.href, + infoHash, + torrentFilename, torrentUrl, - // Use our own torrent name since we proxify torrent requests - torrentFilename: generateTorrentFileName(videoOrPlaylist, resolution), // This is a video file owned by a video or by a streaming playlist videoId, @@ -126,19 +116,17 @@ export function getStreamingPlaylistAttributesFromObject (video: MVideoId, video const files: unknown[] = playlistUrlObject.tag.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] - if (!segmentsSha256UrlObject) { - logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject }) - continue - } - const attribute = { type: VideoStreamingPlaylistType.HLS, playlistFilename: basename(playlistUrlObject.href), playlistUrl: playlistUrlObject.href, - segmentsSha256Filename: basename(segmentsSha256UrlObject.href), - segmentsSha256Url: segmentsSha256UrlObject.href, + segmentsSha256Filename: segmentsSha256UrlObject + ? basename(segmentsSha256UrlObject.href) + : null, + + segmentsSha256Url: segmentsSha256UrlObject?.href ?? null, p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files), p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, @@ -270,3 +258,41 @@ function isAPMagnetUrlObject (url: any): url is ActivityMagnetUrlObject { function isAPHashTagObject (url: any): url is ActivityHashTagObject { return url && url.type === 'Hashtag' } + +function getTorrentRelatedInfo (options: { + videoOrPlaylist: MVideo | MStreamingPlaylistVideo + urls: (ActivityTagObject | ActivityUrlObject)[] + fileUrl: ActivityVideoUrlObject +}) { + const { urls, fileUrl, videoOrPlaylist } = options + + // Fetch associated magnet uri + const magnet = urls.filter(isAPMagnetUrlObject) + .find(u => u.height === fileUrl.height) + + if (!magnet) { + return { + torrentUrl: null, + torrentFilename: null, + infoHash: null + } + } + + const magnetParsed = magnetUriDecode(magnet.href) + if (magnetParsed && isVideoFileInfoHashValid(magnetParsed.infoHash) === false) { + throw new Error('Info hash is not valid in magnet URI ' + magnet.href) + } + + const torrentUrl = Array.isArray(magnetParsed.xs) + ? magnetParsed.xs[0] + : magnetParsed.xs + + return { + torrentUrl, + + // Use our own torrent name since we proxify torrent requests + torrentFilename: generateTorrentFileName(videoOrPlaylist, fileUrl.height), + + infoHash: magnetParsed.infoHash + } +} diff --git a/server/core/lib/activitypub/videos/shared/video-sync-attributes.ts b/server/core/lib/activitypub/videos/shared/video-sync-attributes.ts index a07f519fc..2663b3665 100644 --- a/server/core/lib/activitypub/videos/shared/video-sync-attributes.ts +++ b/server/core/lib/activitypub/videos/shared/video-sync-attributes.ts @@ -56,6 +56,8 @@ async function getRatesCount (type: 'like' | 'dislike', video: MVideo, fetchedVi ? fetchedVideo.likes : fetchedVideo.dislikes + if (!uri) return + logger.info('Sync %s of video %s', type, video.url) const { body } = await fetchAP>(uri) @@ -70,6 +72,7 @@ async function getRatesCount (type: 'like' | 'dislike', video: MVideo, fetchedVi function syncShares (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) { const uri = fetchedVideo.shares + if (!uri) return if (!isSync) { return createJob({ uri, videoId: video.id, type: 'video-shares' }) @@ -84,6 +87,7 @@ function syncShares (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) function syncComments (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) { const uri = fetchedVideo.comments + if (!uri) return if (!isSync) { return createJob({ uri, videoId: video.id, type: 'video-comments' }) diff --git a/server/core/models/video/video-streaming-playlist.ts b/server/core/models/video/video-streaming-playlist.ts index 446c3b810..649ed2716 100644 --- a/server/core/models/video/video-streaming-playlist.ts +++ b/server/core/models/video/video-streaming-playlist.ts @@ -84,7 +84,7 @@ export class VideoStreamingPlaylistModel extends SequelizeModel