Chocobozzz/PeerTube
1
0
Fork 0
mirror of https://github.com/Chocobozzz/PeerTube.git synced 2024-10-30 11:52:24 +03:00
Code Issues Projects Releases Wiki Activity

Merge branch 'webseed'

Browse Source
This commit is contained in:
Chocobozzz 2016-10-07 10:10:00 +02:00
commit aff36eb063
16 changed files with 68 additions and 323 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -14,7 +14,11 @@ uploads
thumbnails
config/production.yaml
ffmpeg
<<<<<<< HEAD
torrents
=======
.tags
*.sublime-project
*.sublime-workspace
torrents/
>>>>>>> master

1
client/src/app/shared/auth/auth.service.ts
View File

@ -123,6 +123,7 @@ export class AuthService {
logout() {
// TODO: make an HTTP request to revoke the tokens
this.user = null;
AuthUser.flush();
this.setStatus(AuthStatus.LoggedOut);

1
client/src/app/videos/video-watch/video-watch.component.ts
View File

@ -68,6 +68,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
ngOnDestroy() {
console.log('Removing video from webtorrent.');
clearInterval(this.torrentInfosInterval);
clearTimeout(this.errorTimer);
this.webTorrentService.remove(this.video.magnetUri);
this.sub.unsubscribe();

1
config/default.yaml
View File

@ -17,6 +17,7 @@ storage:
uploads: 'uploads/'
logs: 'logs/'
thumbnails: 'thumbnails/'
torrents: 'torrents/'
electron:
debug: false

1
config/test-1.yaml
View File

@ -14,3 +14,4 @@ storage:
uploads: 'test1/uploads/'
logs: 'test1/logs/'
thumbnails: 'test1/thumbnails/'
torrents: 'test1/torrents/'

1
config/test-2.yaml
View File

@ -14,3 +14,4 @@ storage:
uploads: 'test2/uploads/'
logs: 'test2/logs/'
thumbnails: 'test2/thumbnails/'
torrents: 'test2/torrents/'

1
config/test-3.yaml
View File

@ -14,3 +14,4 @@ storage:
uploads: 'test3/uploads/'
logs: 'test3/logs/'
thumbnails: 'test3/thumbnails/'
torrents: 'test3/torrents/'

1
config/test-4.yaml
View File

@ -14,3 +14,4 @@ storage:
uploads: 'test4/uploads/'
logs: 'test4/logs/'
thumbnails: 'test4/thumbnails/'
torrents: 'test4/torrents/'

1
config/test-5.yaml
View File

@ -14,3 +14,4 @@ storage:
uploads: 'test5/uploads/'
logs: 'test5/logs/'
thumbnails: 'test5/thumbnails/'
torrents: 'test5/torrents/'

1
config/test-6.yaml
View File

@ -14,3 +14,4 @@ storage:
uploads: 'test6/uploads/'
logs: 'test6/logs/'
thumbnails: 'test6/thumbnails/'
torrents: 'test6/torrents/'

6
package.json
View File

@ -40,27 +40,27 @@
"body-parser": "^1.12.4",
"concurrently": "^2.0.0",
"config": "^1.14.0",
"create-torrent": "^3.24.5",
"debug": "^2.2.0",
"dezalgo": "^1.0.3",
"electron-spawn": "https://github.com/Chocobozzz/electron-spawn",
"express": "^4.12.4",
"express-oauth-server": "https://github.com/Chocobozzz/express-oauth-server",
"express-validator": "^2.11.0",
"fluent-ffmpeg": "^2.1.0",
"js-yaml": "^3.5.4",
"lodash": "^4.11.1",
"magnet-uri": "^5.1.4",
"mkdirp": "^0.5.1",
"mongoose": "^4.0.5",
"morgan": "^1.5.3",
"multer": "^1.1.0",
"node-ipc": "8.9.0",
"openssl-wrapper": "^0.3.4",
"parse-torrent": "^5.8.0",
"password-generator": "^2.0.2",
"request": "^2.57.0",
"request-replay": "^1.0.2",
"scripty": "^1.5.0",
"ursa": "^0.9.1",
"webtorrent": "^0.96.0",
"winston": "^2.1.1",
"ws": "^1.1.1"
},

45
server.js
View File

@ -35,10 +35,7 @@ const installer = require('./server/initializers/installer')
const migrator = require('./server/initializers/migrator')
const mongoose = require('mongoose')
const routes = require('./server/controllers')
const utils = require('./server/helpers/utils')
const webtorrent = require('./server/lib/webtorrent')
const Request = mongoose.model('Request')
const Video = mongoose.model('Video')
// Get configurations
const port = config.get('listen.port')
@ -76,9 +73,16 @@ app.use('/client/*', function (req, res, next) {
res.sendStatus(404)
})
const torrentsPhysicalPath = path.join(__dirname, config.get('storage.torrents'))
app.use(constants.STATIC_PATHS.TORRENTS, express.static(torrentsPhysicalPath, { maxAge: 0 }))
// Uploads path for webseeding
const uploadsPhysicalPath = path.join(__dirname, config.get('storage.uploads'))
app.use(constants.STATIC_PATHS.WEBSEED, express.static(uploadsPhysicalPath, { maxAge: 0 }))
// Thumbnails path for express
const thumbnailsPhysicalPath = path.join(__dirname, config.get('storage.thumbnails'))
app.use(constants.THUMBNAILS_STATIC_PATH, express.static(thumbnailsPhysicalPath, { maxAge: 0 }))
app.use(constants.STATIC_PATHS.THUMBNAILS, express.static(thumbnailsPhysicalPath, { maxAge: 0 }))
// Client application
app.use('/*', function (req, res, next) {
@ -129,33 +133,14 @@ installer.installApplication(function (err) {
migrator.migrate(function (err) {
if (err) throw err
// Create/activate the webtorrent module
webtorrent.create(function () {
function cleanForExit () {
utils.cleanForExit(webtorrent.app)
}
// ----------- Make the server listening -----------
server.listen(port, function () {
// Activate the pool requests
Request.activate()
function exitGracefullyOnSignal () {
process.exit(-1)
}
process.on('exit', cleanForExit)
process.on('SIGINT', exitGracefullyOnSignal)
process.on('SIGTERM', exitGracefullyOnSignal)
// ----------- Make the server listening -----------
server.listen(port, function () {
// Activate the pool requests
Request.activate()
Video.seedAllExisting(function (err) {
if (err) throw err
logger.info('Seeded all the videos')
logger.info('Server listening on port %d', port)
app.emit('ready')
})
})
logger.info('Seeded all the videos')
logger.info('Server listening on port %d', port)
app.emit('ready')
})
})
})

12
server/initializers/constants.js
View File

@ -122,12 +122,16 @@ const RETRY_REQUESTS = 5
// Password encryption
const BCRYPT_SALT_SIZE = 10
// Express static paths (router)
const STATIC_PATHS = {
THUMBNAILS: '/static/thumbnails',
TORRENTS: '/static/torrents/',
WEBSEED: '/static/webseed/'
}
// Videos thumbnail size
const THUMBNAILS_SIZE = '200x110'
// Path for access to thumbnails with express router
const THUMBNAILS_STATIC_PATH = '/static/thumbnails'
const USER_ROLES = {
ADMIN: 'admin',
USER: 'user'
@ -166,8 +170,8 @@ module.exports = {
SEARCHABLE_COLUMNS,
SEEDS_IN_PARALLEL,
SORTABLE_COLUMNS,
STATIC_PATHS,
THUMBNAILS_SIZE,
THUMBNAILS_STATIC_PATH,
USER_ROLES
}

92
server/lib/webtorrent-process.js
View File

@ -1,92 +0,0 @@
'use strict'
const WebTorrent = require('webtorrent')
const ipc = require('node-ipc')
function webtorrent (args) {
if (args.length !== 3) {
throw new Error('Wrong arguments number: ' + args.length + '/3')
}
const host = args[1]
const port = args[2]
const nodeKey = 'webtorrentnode' + port
const processKey = 'webtorrentprocess' + port
ipc.config.silent = true
ipc.config.id = processKey
if (host === 'client' && port === '1') global.WEBTORRENT_ANNOUNCE = []
else global.WEBTORRENT_ANNOUNCE = 'ws://' + host + ':' + port + '/tracker/socket'
const wt = new WebTorrent({ dht: false })
function seed (data) {
const args = data.args
const path = args.path
const _id = data._id
wt.seed(path, { announceList: '' }, function (torrent) {
const toSend = {
magnetUri: torrent.magnetURI
}
ipc.of[nodeKey].emit(nodeKey + '.seedDone.' + _id, toSend)
})
}
function add (data) {
const args = data.args
const magnetUri = args.magnetUri
const _id = data._id
wt.add(magnetUri, function (torrent) {
const toSend = {
files: []
}
torrent.files.forEach(function (file) {
toSend.files.push({ path: file.path })
})
ipc.of[nodeKey].emit(nodeKey + '.addDone.' + _id, toSend)
})
}
function remove (data) {
const args = data.args
const magnetUri = args.magnetUri
const _id = data._id
try {
wt.remove(magnetUri, callback)
} catch (err) {
console.log('Cannot remove the torrent from WebTorrent.')
return callback(null)
}
function callback () {
const toSend = {}
ipc.of[nodeKey].emit(nodeKey + '.removeDone.' + _id, toSend)
}
}
console.log('Configuration: ' + host + ':' + port)
console.log('Connecting to IPC...')
ipc.connectTo(nodeKey, function () {
ipc.of[nodeKey].on(processKey + '.seed', seed)
ipc.of[nodeKey].on(processKey + '.add', add)
ipc.of[nodeKey].on(processKey + '.remove', remove)
ipc.of[nodeKey].emit(processKey + '.ready')
console.log('Ready.')
})
process.on('uncaughtException', function (e) {
ipc.of[nodeKey].emit(processKey + '.exception', { exception: e.toString() })
})
}
// ---------------------------------------------------------------------------
module.exports = webtorrent

160
server/lib/webtorrent.js
View File

@ -1,160 +0,0 @@
'use strict'
const ipc = require('node-ipc')
const pathUtils = require('path')
const spawn = require('electron-spawn')
const constants = require('../initializers/constants')
const logger = require('../helpers/logger')
let host = constants.CONFIG.WEBSERVER.HOST
let port = constants.CONFIG.WEBSERVER.PORT
let nodeKey = 'webtorrentnode' + port
let processKey = 'webtorrentprocess' + port
ipc.config.silent = true
ipc.config.id = nodeKey
const webtorrent = {
add: add,
app: null, // Pid of the app
create: create,
remove: remove,
seed: seed,
silent: false // Useful for beautiful tests
}
function create (options, callback) {
if (typeof options === 'function') {
callback = options
options = {}
}
// Override options
if (options.host) host = options.host
if (options.port) {
port = options.port
nodeKey = 'webtorrentnode' + port
processKey = 'webtorrentprocess' + port
ipc.config.id = nodeKey
}
ipc.serve(function () {
if (!webtorrent.silent) logger.info('IPC server ready.')
// Run a timeout of 30s after which we exit the process
const timeoutWebtorrentProcess = setTimeout(function () {
throw new Error('Timeout : cannot run the webtorrent process. Please ensure you have electron npm package installed with xvfb-run.')
}, 30000)
ipc.server.on(processKey + '.ready', function () {
if (!webtorrent.silent) logger.info('Webtorrent process ready.')
clearTimeout(timeoutWebtorrentProcess)
callback()
})
ipc.server.on(processKey + '.exception', function (data) {
throw new Error('Received exception error from webtorrent process : ' + data.exception)
})
const webtorrentProcess = spawn(pathUtils.join(__dirname, 'webtorrent-process.js'), host, port, { detached: true })
if (constants.CONFIG.ELECTRON.DEBUG === true) {
webtorrentProcess.stderr.on('data', function (data) {
logger.debug('Webtorrent process stderr: ', data.toString())
})
webtorrentProcess.stdout.on('data', function (data) {
logger.debug('Webtorrent process:', data.toString())
})
}
webtorrent.app = webtorrentProcess
})
ipc.server.start()
}
function seed (path, callback) {
const extension = pathUtils.extname(path)
const basename = pathUtils.basename(path, extension)
const data = {
_id: basename,
args: {
path: path
}
}
if (!webtorrent.silent) logger.debug('Node wants to seed %s.', data._id)
// Finish signal
const eventKey = nodeKey + '.seedDone.' + data._id
ipc.server.on(eventKey, function listener (received) {
if (!webtorrent.silent) logger.debug('Process seeded torrent %s.', received.magnetUri)
// This is a fake object, we just use the magnetUri in this project
const torrent = {
magnetURI: received.magnetUri
}
ipc.server.off(eventKey, '*')
callback(torrent)
})
ipc.server.broadcast(processKey + '.seed', data)
}
function add (magnetUri, callback) {
const data = {
_id: magnetUri,
args: {
magnetUri: magnetUri
}
}
if (!webtorrent.silent) logger.debug('Node wants to add ' + data._id)
// Finish signal
const eventKey = nodeKey + '.addDone.' + data._id
ipc.server.on(eventKey, function (received) {
if (!webtorrent.silent) logger.debug('Process added torrent.')
// This is a fake object, we just use the magnetUri in this project
const torrent = {
files: received.files
}
ipc.server.off(eventKey, '*')
callback(torrent)
})
ipc.server.broadcast(processKey + '.add', data)
}
function remove (magnetUri, callback) {
const data = {
_id: magnetUri,
args: {
magnetUri: magnetUri
}
}
if (!webtorrent.silent) logger.debug('Node wants to stop seeding %s.', data._id)
// Finish signal
const eventKey = nodeKey + '.removeDone.' + data._id
ipc.server.on(eventKey, function (received) {
if (!webtorrent.silent) logger.debug('Process removed torrent %s.', data._id)
let err = null
if (received.err) err = received.err
ipc.server.off(eventKey, '*')
callback(err)
})
ipc.server.broadcast(processKey + '.remove', data)
}
// ---------------------------------------------------------------------------
module.exports = webtorrent

63
server/models/video.js
View File

@ -1,10 +1,13 @@
'use strict'
const eachLimit = require('async/eachLimit')
const config = require('config')
const createTorrent = require('create-torrent')
const ffmpeg = require('fluent-ffmpeg')
const fs = require('fs')
const parallel = require('async/parallel')
const parseTorrent = require('parse-torrent')
const pathUtils = require('path')
const magnet = require('magnet-uri')
const mongoose = require('mongoose')
const constants = require('../initializers/constants')
@ -12,7 +15,14 @@ const customVideosValidators = require('../helpers/custom-validators').videos
const logger = require('../helpers/logger')
const modelUtils = require('./utils')
const utils = require('../helpers/utils')
const webtorrent = require('../lib/webtorrent')
const http = config.get('webserver.https') === true ? 'https' : 'http'
const host = config.get('webserver.host')
const port = config.get('webserver.port')
const uploadsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.uploads'))
const thumbnailsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.thumbnails'))
const torrentsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.torrents'))
const webseedBaseUrl = http + '://' + host + ':' + port + constants.STATIC_PATHS.WEBSEED
// ---------------------------------------------------------------------------
@ -61,8 +71,7 @@ VideoSchema.statics = {
listOwnedByAuthor,
listRemotes,
load,
search,
seedAllExisting
search
}
VideoSchema.pre('remove', function (next) {
@ -98,8 +107,21 @@ VideoSchema.pre('save', function (next) {
this.podUrl = constants.CONFIG.WEBSERVER.URL
tasks.push(
// TODO: refractoring
function (callback) {
seed(videoPath, callback)
createTorrent(videoPath, { announceList: [ [ 'ws://' + host + ':' + port + '/tracker/socket' ] ], urlList: [ webseedBaseUrl + video.filename ] }, function (err, torrent) {
if (err) return callback(err)
fs.writeFile(torrentsDir + video.filename + '.torrent', torrent, function (err) {
if (err) return callback(err)
const parsedTorrent = parseTorrent(torrent)
parsedTorrent.xs = video.podUrl + constants.STATIC_PATHS.TORRENTS + video.filename + '.torrent'
video.magnetUri = magnet.encode(parsedTorrent)
callback(null)
})
})
},
function (callback) {
createThumbnail(videoPath, callback)
@ -109,7 +131,6 @@ VideoSchema.pre('save', function (next) {
parallel(tasks, function (err, results) {
if (err) return next(err)
video.magnetUri = results[0].magnetURI
video.thumbnail = results[1]
return next()
@ -144,7 +165,7 @@ function toFormatedJSON () {
author: this.author,
duration: this.duration,
tags: this.tags,
thumbnailPath: constants.THUMBNAILS_STATIC_PATH + '/' + this.thumbnail,
thumbnailPath: constants.STATIC_PATHS.THUMBNAILS + '/' + this.thumbnail,
createdDate: this.createdDate
}
@ -230,17 +251,6 @@ function search (value, field, start, count, sort, callback) {
modelUtils.listForApiWithCount.call(this, query, start, count, sort, callback)
}
function seedAllExisting (callback) {
listOwned.call(this, function (err, videos) {
if (err) return callback(err)
eachLimit(videos, constants.SEEDS_IN_PARALLEL, function (video, callbackEach) {
const videoPath = pathUtils.join(constants.CONFIG.STORAGE.UPLOAD_DIR, video.filename)
seed(videoPath, callbackEach)
}, callback)
})
}
// ---------------------------------------------------------------------------
function removeThumbnail (video, callback) {
@ -253,12 +263,7 @@ function removeFile (video, callback) {
// Maybe the torrent is not seeded, but we catch the error to don't stop the removing process
function removeTorrent (video, callback) {
try {
webtorrent.remove(video.magnetUri, callback)
} catch (err) {
logger.warn('Cannot remove the torrent from WebTorrent', { err: err })
return callback(null)
}
fs.unlink(torrentsDir + video.filename + '.torrent')
}
function createThumbnail (videoPath, callback) {
@ -276,16 +281,6 @@ function createThumbnail (videoPath, callback) {
})
}
function seed (path, callback) {
logger.info('Seeding %s...', path)
webtorrent.seed(path, function (torrent) {
logger.info('%s seeded (%s).', path, torrent.magnetURI)
return callback(null, torrent)
})
}
function generateThumbnailFromBase64 (data, callback) {
// Creating the thumbnail for this remote video
utils.generateRandomString(16, function (err, randomString) {