Support update object storage urls

This commit is contained in:
Chocobozzz 2024-06-05 09:01:40 +02:00
parent 96b9748585
commit 3427330611
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
11 changed files with 280 additions and 8 deletions

View File

@ -47,6 +47,7 @@
"plugin:install": "node ./dist/scripts/plugin/install.js",
"plugin:uninstall": "node ./dist/scripts/plugin/uninstall.js",
"reset-password": "node ./dist/scripts/reset-password.js",
"update-object-storage-url": "LOGGER_LEVEL=warn node ./dist/scripts/update-object-storage-url.js",
"update-host": "node ./dist/scripts/update-host.js",
"regenerate-thumbnails": "node ./dist/scripts/regenerate-thumbnails.js",
"create-import-video-file-job": "node ./dist/scripts/create-import-video-file-job.js",

View File

@ -11,6 +11,7 @@ export interface UserExport {
// In bytes
size: number
fileUrl: string
privateDownloadUrl: string
createdAt: string | Date

View File

@ -10,6 +10,7 @@ export interface VideoSource {
width?: number
height?: number
fileUrl: string
fileDownloadUrl: string
fps?: number

View File

@ -8,3 +8,4 @@ import './prune-storage'
import './regenerate-thumbnails'
import './reset-password'
import './update-host'
import './update-object-storage-url.js'

View File

@ -0,0 +1,131 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { getHLS } from '@peertube/peertube-core-utils'
import { VideoDetails } from '@peertube/peertube-models'
import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils'
import {
ObjectStorageCommand,
PeerTubeServer,
cleanupTests,
createSingleServer,
getRedirectionUrl,
setAccessTokensToServers,
waitJobs
} from '@peertube/peertube-server-commands'
import { expectStartWith } from '@tests/shared/checks.js'
import { expect } from 'chai'
describe('Update object storage URL', function () {
if (areMockObjectStorageTestsDisabled()) return
let server: PeerTubeServer
let uuid: string
const objectStorage = new ObjectStorageCommand()
function runUpdate (from: string, to: string) {
const env = server.cli.getEnv()
const command = `echo y | ${env} npm run update-object-storage-url -- --from "${from}" --to "${to}"`
return server.cli.execWithEnv(command, objectStorage.getDefaultMockConfig())
}
before(async function () {
this.timeout(360000)
server = await createSingleServer(1, objectStorage.getDefaultMockConfig())
await setAccessTokensToServers([ server ])
await objectStorage.prepareDefaultMockBuckets()
await server.config.enableMinimumTranscoding({ keepOriginal: true })
const video = await server.videos.quickUpload({ name: 'video' })
uuid = video.uuid
await waitJobs([ server ])
})
it('Should update video URLs', async function () {
this.timeout(120000)
const check = async (options: {
baseUrl: string
newBaseUrl: string
urlGetter: (video: VideoDetails) => Promise<string[]> | string[]
}) => {
const { baseUrl, newBaseUrl, urlGetter } = options
const oldVideo = await server.videos.get({ id: uuid })
const oldFileUrls = await urlGetter(oldVideo)
for (const url of oldFileUrls) {
expectStartWith(url, baseUrl)
}
await runUpdate(baseUrl, newBaseUrl)
const newVideo = await server.videos.get({ id: uuid })
const shouldBe = oldFileUrls.map(f => f.replace(baseUrl, newBaseUrl))
expect(await urlGetter(newVideo)).to.have.members(shouldBe)
}
await check({
baseUrl: objectStorage.getMockWebVideosBaseUrl(),
newBaseUrl: 'https://web-video.example.com/',
urlGetter: video => video.files.map(f => f.fileUrl)
})
await check({
baseUrl: objectStorage.getMockPlaylistBaseUrl(),
newBaseUrl: 'https://streaming-playlists.example.com/',
urlGetter: video => {
const hls = getHLS(video)
return [
...hls.files.map(f => f.fileUrl),
hls.playlistUrl,
hls.segmentsSha256Url
]
}
})
await check({
baseUrl: objectStorage.getMockOriginalFileBaseUrl(),
newBaseUrl: 'https://original-file.example.com/',
urlGetter: async video => {
const source = await server.videos.getSource({ id: video.uuid })
return [ source.fileUrl ]
}
})
})
it('Should update user export URLs', async function () {
this.timeout(120000)
const user = await server.users.getMyInfo()
await server.userExports.request({ userId: user.id, withVideoFiles: false })
await waitJobs([ server ])
{
const { data } = await server.userExports.list({ userId: user.id })
expectStartWith(data[0].fileUrl, objectStorage.getMockUserExportBaseUrl())
}
await runUpdate(objectStorage.getMockUserExportBaseUrl(), 'https://user-export.example.com/')
{
const { data } = await server.userExports.list({ userId: user.id })
expectStartWith(data[0].fileUrl, 'https://user-export.example.com/')
}
})
after(async function () {
await objectStorage.cleanupMock()
await cleanupTests([ server ])
})
})

View File

@ -27,10 +27,6 @@ export function wordsToRegExp (words: string[]) {
return new RegExp(`(?:\\P{L}|^)(?:${innerRegex})(?=\\P{L}|$)`, 'iu')
}
// ---------------------------------------------------------------------------
// Private
// ---------------------------------------------------------------------------
function escapeForRegex (value: string) {
export function escapeForRegex (value: string) {
return value.replace(/[\\^$.*+?()[\]{}|]/g, '\\$&')
}

View File

@ -31,7 +31,7 @@ export class VideoViewerCounters {
private processingViewerCounters = false
constructor () {
setInterval(() => this.cleanViewerCounters(), VIEW_LIFETIME.VIEWER_COUNTER)
setInterval(() => this.updateVideoViewersCount(), VIEW_LIFETIME.VIEWER_COUNTER)
}
// ---------------------------------------------------------------------------
@ -163,11 +163,13 @@ export class VideoViewerCounters {
return viewer
}
private async cleanViewerCounters () {
private async updateVideoViewersCount () {
if (this.processingViewerCounters) return
this.processingViewerCounters = true
if (!isTestOrDevInstance()) logger.info('Cleaning video viewers.', lTags())
if (!isTestOrDevInstance()) {
logger.debug('Updating video viewer counters.', lTags())
}
try {
for (const videoId of this.viewersPerVideo.keys()) {

View File

@ -219,6 +219,7 @@ export class UserExportModel extends SequelizeModel<UserExportModel> {
size: this.size,
fileUrl: this.fileUrl,
privateDownloadUrl: this.getFileDownloadUrl(),
createdAt: this.createdAt.toISOString(),
expiresOn: new Date(this.createdAt.getTime() + CONFIG.EXPORT.USERS.EXPORT_EXPIRATION).toISOString()

View File

@ -125,6 +125,8 @@ export class VideoSourceModel extends SequelizeModel<VideoSourceModel> {
return {
filename: this.inputFilename,
inputFilename: this.inputFilename,
fileUrl: this.fileUrl,
fileDownloadUrl: this.getFileDownloadUrl(),
resolution: {

View File

@ -0,0 +1,118 @@
/* eslint-disable max-len */
import { InvalidArgumentError, createCommand } from '@commander-js/extra-typings'
import { FileStorage } from '@peertube/peertube-models'
import { escapeForRegex } from '@server/helpers/regexp.js'
import { initDatabaseModels, sequelizeTypescript } from '@server/initializers/database.js'
import { QueryTypes } from 'sequelize'
import prompt from 'prompt'
const program = createCommand()
.description('Update PeerTube object file URLs after an object storage migration.')
.requiredOption('-f, --from <url>', 'Previous object storage base URL', parseUrl)
.requiredOption('-t, --to <url>', 'New object storage base URL', parseUrl)
.parse(process.argv)
const options = program.opts()
run()
.then(() => process.exit(0))
.catch(err => {
console.error(err)
process.exit(-1)
})
async function run () {
await initDatabaseModels(true)
const fromRegexp = `^${escapeForRegex(options.from)}`
const to = options.to
const replacements = { fromRegexp, to, storage: FileStorage.OBJECT_STORAGE }
// Candidates
{
const queries = [
`SELECT COUNT(*) AS "c", 'videoFile->fileUrl: ' || COUNT(*) AS "t" FROM "videoFile" WHERE "fileUrl" ~ :fromRegexp AND "storage" = :storage`,
`SELECT COUNT(*) AS "c", 'videoStreamingPlaylist->playlistUrl: ' || COUNT(*) AS "t" FROM "videoStreamingPlaylist" WHERE "playlistUrl" ~ :fromRegexp AND "storage" = :storage`,
`SELECT COUNT(*) AS "c", 'videoStreamingPlaylist->segmentsSha256Url: ' || COUNT(*) AS "t" FROM "videoStreamingPlaylist" WHERE "segmentsSha256Url" ~ :fromRegexp AND "storage" = :storage`,
`SELECT COUNT(*) AS "c", 'userExport->fileUrl: ' || COUNT(*) AS "t" FROM "userExport" WHERE "fileUrl" ~ :fromRegexp AND "storage" = :storage`,
`SELECT COUNT(*) AS "c", 'videoSource->fileUrl: ' || COUNT(*) AS "t" FROM "videoSource" WHERE "fileUrl" ~ :fromRegexp AND "storage" = :storage`
]
let hasResults = false
console.log('Candidate URLs to update:')
for (const query of queries) {
const [ row ] = await sequelizeTypescript.query(query, { replacements, type: QueryTypes.SELECT as QueryTypes.SELECT })
if (row['c'] !== 0) hasResults = true
console.log(` ${row['t']}`)
}
console.log('\n')
if (!hasResults) {
console.log('No candidate URLs found, exiting.')
process.exit(0)
}
}
const res = await askConfirmation()
if (res !== true) {
console.log('Exiting without updating URLs.')
process.exit(0)
}
// Execute
{
const queries = [
`UPDATE "videoFile" SET "fileUrl" = regexp_replace("fileUrl", :fromRegexp, :to) WHERE "storage" = :storage`,
`UPDATE "videoStreamingPlaylist" SET "playlistUrl" = regexp_replace("playlistUrl", :fromRegexp, :to) WHERE "storage" = :storage`,
`UPDATE "videoStreamingPlaylist" SET "segmentsSha256Url" = regexp_replace("segmentsSha256Url", :fromRegexp, :to) WHERE "storage" = :storage`,
`UPDATE "userExport" SET "fileUrl" = regexp_replace("fileUrl", :fromRegexp, :to) WHERE "storage" = :storage`,
`UPDATE "videoSource" SET "fileUrl" = regexp_replace("fileUrl", :fromRegexp, :to) WHERE "storage" = :storage`
]
for (const query of queries) {
await sequelizeTypescript.query(query, { replacements })
}
console.log('URLs updated.')
}
}
function parseUrl (value: string) {
if (!value || /^https?:\/\//.test(value) !== true) {
throw new InvalidArgumentError('Must be a valid URL (starting with http:// or https://).')
}
return value
}
async function askConfirmation () {
return new Promise((res, rej) => {
prompt.start()
const schema = {
properties: {
confirm: {
type: 'string',
description: 'These URLs can be updated, but please check your backups first (bugs happen).' +
' Notice PeerTube must have been stopped when your ran this script.' +
' Can we update these URLs? (y/n)',
default: 'n',
validator: /y[es]*|n[o]?/,
warning: 'Must respond yes or no',
required: true
}
}
}
prompt.get(schema, function (err, result) {
if (err) return rej(err)
return res(result.confirm?.match(/y/) !== null)
})
})
}

View File

@ -472,6 +472,24 @@ docker compose exec -u peertube peertube npm run create-move-video-storage-job -
:::
### Update object storage URLs
**PeerTube >= 6.2**
Use this script after you migrated to another object storage provider so PeerTube updates its internal object URLs.
::: code-group
```bash [Classic installation]
cd /var/www/peertube/peertube-latest
sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run update-object-storage-url -- --from 'https://region.old-s3-provider.example.com' --to 'https://region.new-s3-provider.example.com'
```
```bash [Docker]
cd /var/www/peertube-docker
docker compose exec -u peertube peertube npm run update-object-storage-url -- --from 'https://region.old-s3-provider.example.com' --to 'https://region.new-s3-provider.example.com'
```
### Generate storyboard
**PeerTube >= 6.0**