mirror of
https://github.com/TryGhost/Ghost.git
synced 2024-11-28 14:03:48 +03:00
🔒 Improved validation of fetched urls and responses in oembed endpoint
no issue - prevent oembed fetching from accessing IP addresses or localhost domains - prevent oembed endpoint from passing through fetched responses as-is - reject any fetched data that does not validate against the oembed spec - strip any unknown properties from the oembed response before returning Credits: Nick Mykhailyshyn
This commit is contained in:
Kevin Ansfield
parent
afb3177e5e
commit
4773939670
@ -3,6 +3,7 @@ const {extract, hasProvider} = require('oembed-parser');
|
||||
const Promise = require('bluebird');
|
||||
const request = require('../../lib/request');
|
||||
const cheerio = require('cheerio');
|
||||
const _ = require('lodash');
|
||||
|
||||
async function fetchBookmarkData(url, html) {
|
||||
const metascraper = require('metascraper')([
|
||||
@ -74,10 +75,6 @@ const findUrlWithProvider = (url) => {
|
||||
return {url, provider};
|
||||
};
|
||||
|
||||
const getOembedUrlFromHTML = (html) => {
|
||||
return cheerio('link[type="application/json+oembed"]', html).attr('href');
|
||||
};
|
||||
|
||||
function unknownProvider(url) {
|
||||
return Promise.reject(new common.errors.ValidationError({
|
||||
message: common.i18n.t('errors.api.oembed.unknownProvider'),
|
||||
@ -93,12 +90,40 @@ function knownProvider(url) {
|
||||
});
|
||||
}
|
||||
|
||||
function fetchOembedData(url) {
|
||||
let provider;
|
||||
({url, provider} = findUrlWithProvider(url));
|
||||
function isIpOrLocalhost(url) {
|
||||
try {
|
||||
const IPV4_REGEX = /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/;
|
||||
const IPV6_REGEX = /:/; // fqdns will not have colons
|
||||
const HTTP_REGEX = /^https?:/i;
|
||||
|
||||
let {protocol, hostname} = new URL(url);
|
||||
|
||||
if (!HTTP_REGEX.test(protocol) || hostname === 'localhost' || IPV4_REGEX.test(hostname) || IPV6_REGEX.test(hostname)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
} catch (e) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
function fetchOembedData(_url) {
|
||||
// parse the url then validate the protocol and host to make sure it's
|
||||
// http(s) and not an IP address or localhost to avoid potential access to
|
||||
// internal network endpoints
|
||||
if (isIpOrLocalhost(_url)) {
|
||||
return unknownProvider();
|
||||
}
|
||||
|
||||
// check against known oembed list
|
||||
let {url, provider} = findUrlWithProvider(_url);
|
||||
if (provider) {
|
||||
return knownProvider(url);
|
||||
}
|
||||
|
||||
// url not in oembed list so fetch it in case it's a redirect or has a
|
||||
// <link rel="alternate" type="application/json+oembed"> element
|
||||
return request(url, {
|
||||
method: 'GET',
|
||||
timeout: 2 * 1000,
|
||||
@ -107,19 +132,70 @@ function fetchOembedData(url) {
|
||||
'user-agent': 'Ghost(https://github.com/TryGhost/Ghost)'
|
||||
}
|
||||
}).then((response) => {
|
||||
// url changed after fetch, see if we were redirected to a known oembed
|
||||
if (response.url !== url) {
|
||||
({url, provider} = findUrlWithProvider(response.url));
|
||||
if (provider) {
|
||||
return knownProvider(url);
|
||||
}
|
||||
}
|
||||
if (provider) {
|
||||
return knownProvider(url);
|
||||
|
||||
// check for <link rel="alternate" type="application/json+oembed"> element
|
||||
let oembedUrl;
|
||||
try {
|
||||
oembedUrl = cheerio('link[type="application/json+oembed"]', response.body).attr('href');
|
||||
} catch (e) {
|
||||
return unknownProvider(url);
|
||||
}
|
||||
const oembedUrl = getOembedUrlFromHTML(response.body);
|
||||
|
||||
if (oembedUrl) {
|
||||
// make sure the linked url is not an ip address or localhost
|
||||
if (isIpOrLocalhost(oembedUrl)) {
|
||||
return unknownProvider(oembedUrl);
|
||||
}
|
||||
|
||||
// fetch oembed response from embedded rel="alternate" url
|
||||
return request(oembedUrl, {
|
||||
method: 'GET',
|
||||
json: true
|
||||
}).then((response) => {
|
||||
return response.body;
|
||||
// validate the fetched json against the oembed spec to avoid
|
||||
// leaking non-oembed responses
|
||||
const body = response.body;
|
||||
const hasRequiredFields = body.type && body.version;
|
||||
const hasValidType = ['photo', 'video', 'link', 'rich'].includes(body.type);
|
||||
|
||||
if (hasRequiredFields && hasValidType) {
|
||||
// extract known oembed fields from the response to limit leaking of unrecognised data
|
||||
const knownFields = [
|
||||
'type',
|
||||
'version',
|
||||
'html',
|
||||
'url',
|
||||
'title',
|
||||
'width',
|
||||
'height',
|
||||
'author_name',
|
||||
'author_url',
|
||||
'provider_name',
|
||||
'provider_url',
|
||||
'thumbnail_url',
|
||||
'thumbnail_width',
|
||||
'thumbnail_height'
|
||||
];
|
||||
const oembed = _.pick(body, knownFields);
|
||||
|
||||
// ensure we have required data for certain types
|
||||
if (oembed.type === 'photo' && !oembed.url) {
|
||||
return;
|
||||
}
|
||||
if ((oembed.type === 'video' || oembed.type === 'rich') && (!oembed.html || !oembed.width || !oembed.height)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// return the extracted object, don't pass through the response body
|
||||
return oembed;
|
||||
}
|
||||
}).catch(() => {});
|
||||
}
|
||||
});
|
||||
|
||||
@ -56,4 +56,278 @@ describe('Oembed API', function () {
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
describe('with unknown provider', function () {
|
||||
it('fetches url and follows <link rel="alternate">', function (done) {
|
||||
const pageMock = nock('http://test.com')
|
||||
.get('/')
|
||||
.reply(200, '<html><head><link rel="alternate" type="application/json+oembed" href="http://test.com/oembed"></head></html>');
|
||||
|
||||
const oembedMock = nock('http://test.com')
|
||||
.get('/oembed')
|
||||
.reply(200, {
|
||||
version: '1.0',
|
||||
type: 'link'
|
||||
});
|
||||
|
||||
const url = encodeURIComponent('http://test.com');
|
||||
request.get(localUtils.API.getApiQuery(`oembed/?url=${url}`))
|
||||
.set('Origin', config.get('url'))
|
||||
.expect('Content-Type', /json/)
|
||||
.expect('Cache-Control', testUtils.cacheRules.private)
|
||||
.expect(200)
|
||||
.end(function (err, res) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
pageMock.isDone().should.be.true();
|
||||
oembedMock.isDone().should.be.true();
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects invalid oembed responses', function (done) {
|
||||
const pageMock = nock('http://test.com')
|
||||
.get('/')
|
||||
.reply(200, '<html><head><link rel="alternate" type="application/json+oembed" href="http://test.com/oembed"></head></html>');
|
||||
|
||||
const oembedMock = nock('http://test.com')
|
||||
.get('/oembed')
|
||||
.reply(200, {
|
||||
version: '1.0',
|
||||
html: 'test'
|
||||
});
|
||||
|
||||
const url = encodeURIComponent('http://test.com');
|
||||
request.get(localUtils.API.getApiQuery(`oembed/?url=${url}`))
|
||||
.set('Origin', config.get('url'))
|
||||
.expect('Content-Type', /json/)
|
||||
.expect('Cache-Control', testUtils.cacheRules.private)
|
||||
.expect(422)
|
||||
.end(function (err, res) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
pageMock.isDone().should.be.true();
|
||||
oembedMock.isDone().should.be.true();
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects unknown oembed types', function (done) {
|
||||
const pageMock = nock('http://test.com')
|
||||
.get('/')
|
||||
.reply(200, '<html><head><link rel="alternate" type="application/json+oembed" href="http://test.com/oembed"></head></html>');
|
||||
|
||||
const oembedMock = nock('http://test.com')
|
||||
.get('/oembed')
|
||||
.reply(200, {
|
||||
version: '1.0',
|
||||
type: 'unknown'
|
||||
});
|
||||
|
||||
const url = encodeURIComponent('http://test.com');
|
||||
request.get(localUtils.API.getApiQuery(`oembed/?url=${url}`))
|
||||
.set('Origin', config.get('url'))
|
||||
.expect('Content-Type', /json/)
|
||||
.expect('Cache-Control', testUtils.cacheRules.private)
|
||||
.expect(422)
|
||||
.end(function (err, res) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
pageMock.isDone().should.be.true();
|
||||
oembedMock.isDone().should.be.true();
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects invalid photo responses', function (done) {
|
||||
const pageMock = nock('http://test.com')
|
||||
.get('/')
|
||||
.reply(200, '<html><head><link rel="alternate" type="application/json+oembed" href="http://test.com/oembed"></head></html>');
|
||||
|
||||
const oembedMock = nock('http://test.com')
|
||||
.get('/oembed')
|
||||
.reply(200, {
|
||||
// no `url` field
|
||||
version: '1.0',
|
||||
type: 'photo',
|
||||
thumbnail_url: 'https://test.com/thumbnail.jpg'
|
||||
});
|
||||
|
||||
const url = encodeURIComponent('http://test.com');
|
||||
request.get(localUtils.API.getApiQuery(`oembed/?url=${url}`))
|
||||
.set('Origin', config.get('url'))
|
||||
.expect('Content-Type', /json/)
|
||||
.expect('Cache-Control', testUtils.cacheRules.private)
|
||||
.expect(422)
|
||||
.end(function (err, res) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
pageMock.isDone().should.be.true();
|
||||
oembedMock.isDone().should.be.true();
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects invalid video responses', function (done) {
|
||||
const pageMock = nock('http://test.com')
|
||||
.get('/')
|
||||
.reply(200, '<html><head><link rel="alternate" type="application/json+oembed" href="http://test.com/oembed"></head></html>');
|
||||
|
||||
const oembedMock = nock('http://test.com')
|
||||
.get('/oembed')
|
||||
.reply(200, {
|
||||
// no `html` field
|
||||
version: '1.0',
|
||||
type: 'video',
|
||||
thumbnail_url: 'https://test.com/thumbnail.jpg'
|
||||
});
|
||||
|
||||
const url = encodeURIComponent('http://test.com');
|
||||
request.get(localUtils.API.getApiQuery(`oembed/?url=${url}`))
|
||||
.set('Origin', config.get('url'))
|
||||
.expect('Content-Type', /json/)
|
||||
.expect('Cache-Control', testUtils.cacheRules.private)
|
||||
.expect(422)
|
||||
.end(function (err, res) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
pageMock.isDone().should.be.true();
|
||||
oembedMock.isDone().should.be.true();
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('strips unknown response fields', function (done) {
|
||||
const pageMock = nock('http://test.com')
|
||||
.get('/')
|
||||
.reply(200, '<html><head><link rel="alternate" type="application/json+oembed" href="http://test.com/oembed"></head></html>');
|
||||
|
||||
const oembedMock = nock('http://test.com')
|
||||
.get('/oembed')
|
||||
.reply(200, {
|
||||
version: '1.0',
|
||||
type: 'video',
|
||||
html: '<p>Test</p>',
|
||||
width: 200,
|
||||
height: 100,
|
||||
unknown: 'test'
|
||||
});
|
||||
|
||||
const url = encodeURIComponent('http://test.com');
|
||||
request.get(localUtils.API.getApiQuery(`oembed/?url=${url}`))
|
||||
.set('Origin', config.get('url'))
|
||||
.expect('Content-Type', /json/)
|
||||
.expect('Cache-Control', testUtils.cacheRules.private)
|
||||
.expect(200)
|
||||
.end(function (err, res) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
pageMock.isDone().should.be.true();
|
||||
oembedMock.isDone().should.be.true();
|
||||
|
||||
res.body.should.deepEqual({
|
||||
version: '1.0',
|
||||
type: 'video',
|
||||
html: '<p>Test</p>',
|
||||
width: 200,
|
||||
height: 100
|
||||
});
|
||||
should.not.exist(res.body.unknown);
|
||||
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('skips fetching IPv4 addresses', function (done) {
|
||||
const pageMock = nock('http://test.com')
|
||||
.get('/')
|
||||
.reply(200, '<html><head><link rel="alternate" type="application/json+oembed" href="http://192.168.0.1/oembed"></head></html>');
|
||||
|
||||
const oembedMock = nock('http://192.168.0.1')
|
||||
.get('/oembed')
|
||||
.reply(200, {
|
||||
version: '1.0',
|
||||
type: 'link'
|
||||
});
|
||||
|
||||
const url = encodeURIComponent('http://test.com');
|
||||
request.get(localUtils.API.getApiQuery(`oembed/?url=${url}`))
|
||||
.set('Origin', config.get('url'))
|
||||
.expect('Content-Type', /json/)
|
||||
.expect('Cache-Control', testUtils.cacheRules.private)
|
||||
.expect(422)
|
||||
.end(function (err, res) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
pageMock.isDone().should.be.true();
|
||||
oembedMock.isDone().should.be.false();
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('skips fetching IPv6 addresses', function (done) {
|
||||
const pageMock = nock('http://test.com')
|
||||
.get('/')
|
||||
.reply(200, '<html><head><link rel="alternate" type="application/json+oembed" href="http://[2607:f0d0:1002:51::4]:9999/oembed"></head></html>');
|
||||
|
||||
const oembedMock = nock('http://[2607:f0d0:1002:51::4]:9999')
|
||||
.get('/oembed')
|
||||
.reply(200, {
|
||||
version: '1.0',
|
||||
type: 'link'
|
||||
});
|
||||
|
||||
const url = encodeURIComponent('http://test.com');
|
||||
request.get(localUtils.API.getApiQuery(`oembed/?url=${url}`))
|
||||
.set('Origin', config.get('url'))
|
||||
.expect('Content-Type', /json/)
|
||||
.expect('Cache-Control', testUtils.cacheRules.private)
|
||||
.expect(422)
|
||||
.end(function (err, res) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
pageMock.isDone().should.be.true();
|
||||
oembedMock.isDone().should.be.false();
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('skips fetching localhost', function (done) {
|
||||
const pageMock = nock('http://test.com')
|
||||
.get('/')
|
||||
.reply(200, '<html><head><link rel="alternate" type="application/json+oembed" href="http://localhost:9999/oembed"></head></html>');
|
||||
|
||||
const oembedMock = nock('http://localhost:9999')
|
||||
.get('/oembed')
|
||||
.reply(200, {
|
||||
// no `html` field
|
||||
version: '1.0',
|
||||
type: 'video',
|
||||
thumbnail_url: 'https://test.com/thumbnail.jpg'
|
||||
});
|
||||
|
||||
const url = encodeURIComponent('http://test.com');
|
||||
request.get(localUtils.API.getApiQuery(`oembed/?url=${url}`))
|
||||
.set('Origin', config.get('url'))
|
||||
.expect('Content-Type', /json/)
|
||||
.expect('Cache-Control', testUtils.cacheRules.private)
|
||||
.expect(422)
|
||||
.end(function (err, res) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
pageMock.isDone().should.be.true();
|
||||
oembedMock.isDone().should.be.false();
|
||||
done();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Loading…
Reference in New Issue
Block a user