Support proxies for PeerTube (#4346)

* Updated with latest code

* Updated Support proxies for PeerTube

* Support Proxies for PeerTube (Updated with change request)

* Cleanup proxy PR

Co-authored-by: Chocobozzz <me@florianbigard.com>
This commit is contained in:
smilekison 2021-08-25 06:08:37 -07:00 committed by GitHub
parent fdec51e384
commit 8729a87024
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 191 additions and 3 deletions

View File

@ -99,6 +99,7 @@
"fs-extra": "^10.0.0",
"got": "^11.8.2",
"helmet": "^4.1.0",
"hpagent": "^0.1.2",
"http-problem-details": "^0.1.5",
"http-signature": "1.3.5",
"ip-anonymize": "^0.1.0",
@ -199,6 +200,7 @@
"marked-man": "^0.7.0",
"mocha": "^9.0.0",
"nodemon": "^2.0.1",
"proxy": "^1.0.2",
"socket.io-client": "^4.0.1",
"source-map-support": "^0.5.0",
"supertest": "^6.0.1",

14
server/helpers/proxy.ts Normal file
View File

@ -0,0 +1,14 @@
function getProxy () {
return process.env.HTTPS_PROXY ||
process.env.HTTP_PROXY ||
undefined
}
function isProxyEnabled () {
return !!getProxy()
}
export {
getProxy,
isProxyEnabled
}

View File

@ -1,19 +1,21 @@
import { createWriteStream, remove } from 'fs-extra'
import got, { CancelableRequest, Options as GotOptions, RequestError } from 'got'
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'
import { join } from 'path'
import { CONFIG } from '../initializers/config'
import { ACTIVITY_PUB, PEERTUBE_VERSION, REQUEST_TIMEOUT, WEBSERVER } from '../initializers/constants'
import { pipelinePromise } from './core-utils'
import { processImage } from './image-utils'
import { logger } from './logger'
import { getProxy, isProxyEnabled } from './proxy'
const httpSignature = require('http-signature')
export interface PeerTubeRequestError extends Error {
statusCode?: number
responseBody?: any
}
const httpSignature = require('http-signature')
type PeerTubeRequestOptions = {
activityPub?: boolean
bodyKBLimit?: number // 1MB
@ -29,6 +31,8 @@ type PeerTubeRequestOptions = {
} & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'>
const peertubeGot = got.extend({
...getAgent(),
headers: {
'user-agent': getUserAgent()
},
@ -153,6 +157,30 @@ async function downloadImage (url: string, destDir: string, destName: string, si
}
}
function getAgent () {
if (!isProxyEnabled()) return {}
const proxy = getProxy()
logger.info('Using proxy %s.', proxy)
const proxyAgentOptions = {
keepAlive: true,
keepAliveMsecs: 1000,
maxSockets: 256,
maxFreeSockets: 256,
scheduling: 'lifo' as 'lifo',
proxy
}
return {
agent: {
http: new HttpProxyAgent(proxyAgentOptions),
https: new HttpsProxyAgent(proxyAgentOptions)
}
}
}
function getUserAgent () {
return `PeerTube/${PEERTUBE_VERSION} (+${WEBSERVER.URL})`
}

View File

@ -15,3 +15,4 @@ import './stats'
import './tracker'
import './no-client'
import './plugins'
import './proxy'

View File

@ -0,0 +1,72 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import 'mocha'
import * as chai from 'chai'
import { cleanupTests, createMultipleServers, doubleFollow, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@shared/extra-utils'
import { MockProxy } from '@shared/extra-utils/mock-servers/mock-proxy'
const expect = chai.expect
describe('Test proxy', function () {
let servers: PeerTubeServer[] = []
let proxy: MockProxy
const goodEnv = { HTTP_PROXY: '' }
const badEnv = { HTTP_PROXY: 'http://localhost:9000' }
before(async function () {
this.timeout(120000)
proxy = new MockProxy()
const proxyPort = await proxy.initialize()
servers = await createMultipleServers(2)
goodEnv.HTTP_PROXY = 'http://localhost:' + proxyPort
await setAccessTokensToServers(servers)
await doubleFollow(servers[0], servers[1])
})
it('Should succeed federation with the appropriate proxy config', async function () {
await servers[0].kill()
await servers[0].run({}, { env: goodEnv })
await servers[0].videos.quickUpload({ name: 'video 1' })
await waitJobs(servers)
for (const server of servers) {
const { total, data } = await server.videos.list()
expect(total).to.equal(1)
expect(data).to.have.lengthOf(1)
}
})
it('Should fail federation with a wrong proxy config', async function () {
await servers[0].kill()
await servers[0].run({}, { env: badEnv })
await servers[0].videos.quickUpload({ name: 'video 2' })
await waitJobs(servers)
{
const { total, data } = await servers[0].videos.list()
expect(total).to.equal(2)
expect(data).to.have.lengthOf(2)
}
{
const { total, data } = await servers[1].videos.list()
expect(total).to.equal(1)
expect(data).to.have.lengthOf(1)
}
})
after(async function () {
proxy.terminate()
await cleanupTests(servers)
})
})

View File

@ -0,0 +1,27 @@
import { createServer, Server } from 'http'
import * as proxy from 'proxy'
import { randomInt } from '@shared/core-utils'
class MockProxy {
private server: Server
initialize () {
return new Promise<number>(res => {
const port = 42501 + randomInt(1, 100)
this.server = proxy(createServer())
this.server.listen(port, () => res(port))
})
}
terminate () {
if (this.server) this.server.close()
}
}
// ---------------------------------------------------------------------------
export {
MockProxy
}

View File

@ -2002,6 +2002,16 @@ argparse@^2.0.1:
resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38"
integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==
args@5.0.1:
version "5.0.1"
resolved "https://registry.yarnpkg.com/args/-/args-5.0.1.tgz#4bf298df90a4799a09521362c579278cc2fdd761"
integrity sha512-1kqmFCFsPffavQFGt8OxJdIcETti99kySRUPMpOhaGjL6mRJn8HFU1OxKY5bMqfZKUwTQc1mZkAjmGYaVOHFtQ==
dependencies:
camelcase "5.0.0"
chalk "2.4.2"
leven "2.1.0"
mri "1.1.4"
array-differ@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/array-differ/-/array-differ-3.0.0.tgz#3cbb3d0f316810eafcc47624734237d6aee4ae6b"
@ -2200,6 +2210,11 @@ basic-auth-connect@^1.0.0:
resolved "https://registry.yarnpkg.com/basic-auth-connect/-/basic-auth-connect-1.0.0.tgz#fdb0b43962ca7b40456a7c2bb48fe173da2d2122"
integrity sha1-/bC0OWLKe0BFanwrtI/hc9otISI=
basic-auth-parser@0.0.2:
version "0.0.2"
resolved "https://registry.yarnpkg.com/basic-auth-parser/-/basic-auth-parser-0.0.2.tgz#ce9e71a77f23c1279eecd2659b2a46244c156e41"
integrity sha1-zp5xp38jwSee7NJlmypGJEwVbkE=
basic-auth@2.0.1, basic-auth@~2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/basic-auth/-/basic-auth-2.0.1.tgz#b998279bf47ce38344b4f3cf916d4679bbf51e3a"
@ -2616,6 +2631,11 @@ camelcase-keys@^4.0.0:
map-obj "^2.0.0"
quick-lru "^1.0.0"
camelcase@5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.0.0.tgz#03295527d58bd3cd4aa75363f35b2e8d97be2f42"
integrity sha512-faqwZqnWxbxn+F1d399ygeamQNy3lPp/H9H6rNrqYh4FSVCtcY+3cub1MxA8o9mDd55mM8Aghuu/kuyYA6VTsA==
camelcase@^4.1.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-4.1.0.tgz#d545635be1e33c542649c69173e5de6acfae34dd"
@ -2663,7 +2683,7 @@ chai@^4.1.1:
pathval "^1.1.1"
type-detect "^4.0.5"
chalk@^2.0.0, chalk@^2.4.2:
chalk@2.4.2, chalk@^2.0.0, chalk@^2.4.2:
version "2.4.2"
resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424"
integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==
@ -4692,6 +4712,11 @@ hosted-git-info@^2.1.4:
resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.9.tgz#dffc0bf9a21c02209090f2aa69429e1414daf3f9"
integrity sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==
hpagent@^0.1.2:
version "0.1.2"
resolved "https://registry.yarnpkg.com/hpagent/-/hpagent-0.1.2.tgz#cab39c66d4df2d4377dbd212295d878deb9bdaa9"
integrity sha512-ePqFXHtSQWAFXYmj+JtOTHr84iNrII4/QRlAAPPE+zqnKy4xJo7Ie1Y4kC7AdB+LxLxSTTzBMASsEcy0q8YyvQ==
html-to-text@8.0.0:
version "8.0.0"
resolved "https://registry.yarnpkg.com/html-to-text/-/html-to-text-8.0.0.tgz#5848681a5a38d657a7bb58cf5006d1c29fe64ce3"
@ -5548,6 +5573,11 @@ latest-version@^5.0.0:
dependencies:
package-json "^6.3.0"
leven@2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/leven/-/leven-2.1.0.tgz#c2e7a9f772094dee9d34202ae8acce4687875580"
integrity sha1-wuep93IJTe6dNCAq6KzORoeHVYA=
levn@^0.4.1:
version "0.4.1"
resolved "https://registry.yarnpkg.com/levn/-/levn-0.4.1.tgz#ae4562c007473b932a6200d403268dd2fffc6ade"
@ -6196,6 +6226,11 @@ mp4-stream@^3.0.0:
queue-microtask "^1.2.2"
readable-stream "^3.0.6"
mri@1.1.4:
version "1.1.4"
resolved "https://registry.yarnpkg.com/mri/-/mri-1.1.4.tgz#7cb1dd1b9b40905f1fac053abe25b6720f44744a"
integrity sha512-6y7IjGPm8AzlvoUrwAaw1tLnUBudaS3752vcd8JtrpGGQn+rXIe63LFVHm/YMwtqAuh+LJPCFdlLYPWM1nYn6w==
ms@2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
@ -7131,6 +7166,15 @@ proxy-addr@~2.0.5:
forwarded "0.2.0"
ipaddr.js "1.9.1"
proxy@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/proxy/-/proxy-1.0.2.tgz#e0cfbe11c0a7a8b238fd2d7134de4e2867578e7f"
integrity sha512-KNac2ueWRpjbUh77OAFPZuNdfEqNynm9DD4xHT14CccGpW8wKZwEkN0yjlb7X9G9Z9F55N0Q+1z+WfgAhwYdzQ==
dependencies:
args "5.0.1"
basic-auth-parser "0.0.2"
debug "^4.1.1"
pseudomap@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3"