Added BookshelfMentionRepository implementation (#16156)

fixes https://github.com/TryGhost/Team/issues/2418

This stores the received webmentions in the database.

Co-authored-by: Simon Backx <simon@ghost.org>
This commit is contained in:
Fabien 'egg' O'Carroll 2023-01-20 20:32:50 +07:00 committed by GitHub
parent c7230f1858
commit 33ebe971f8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 255 additions and 31 deletions

View File

@ -25,8 +25,9 @@ module.exports = {
response: { response: {
format: 'plain' format: 'plain'
}, },
query(frame) { async query(frame) {
return mentions.controller.receive(frame); await mentions.controller.receive(frame);
return null;
} }
} }
}; };

View File

@ -0,0 +1,9 @@
const ghostBookshelf = require('./base');
const Mention = ghostBookshelf.Model.extend({
tableName: 'mentions'
});
module.exports = {
Mention: ghostBookshelf.model('Mention', Mention)
};

View File

@ -0,0 +1,117 @@
const {Mention} = require('@tryghost/webmentions');
const logging = require('@tryghost/logging');
/**
* @typedef {import('@tryghost/webmentions/lib/MentionsAPI').IMentionRepository} IMentionRepository
*/
/**
* @template Model
* @typedef {import('@tryghost/webmentions/lib/MentionsAPI').Page<Model>} Page
*/
/**
* @typedef {import('@tryghost/webmentions/lib/MentionsAPI').GetPageOptions} GetPageOptions
*/
/**
* @implements {IMentionRepository}
*/
module.exports = class BookshelfMentionRepository {
/** @type {Object} */
#MentionModel;
/**
* @param {object} deps
* @param {object} deps.MentionModel Bookshelf Model
*/
constructor(deps) {
this.#MentionModel = deps.MentionModel;
}
#modelToMention(model) {
let payload;
try {
payload = JSON.parse(model.get('payload'));
} catch (err) {
logging.error(err);
payload = {};
}
return Mention.create({
id: model.get('id'),
source: model.get('source'),
target: model.get('target'),
timestamp: model.get('created_at'),
payload,
resourceId: model.get('resource_id'),
sourceTitle: model.get('source_title'),
sourceSiteTitle: model.get('source_site_title'),
sourceAuthor: model.get('source_author'),
sourceExcerpt: model.get('source_excerpt'),
sourceFavicon: model.get('source_favicon'),
sourceFeaturedImaged: model.get('source_featured_image')
});
}
/**
* @param {GetPageOptions} options
* @returns {Promise<Page<import('@tryghost/webmentions/lib/Mention')>>}
*/
async getPage(options) {
const page = await this.#MentionModel.findPage(options);
return {
data: await Promise.all(page.data.map(model => this.#modelToMention(model))),
meta: page.meta
};
}
/**
* @param {URL} source
* @param {URL} target
* @returns {Promise<import('@tryghost/webmentions/lib/Mention')|null>}
*/
async getBySourceAndTarget(source, target) {
const model = await this.#MentionModel.findOne({
source: source.href,
target: target.href
}, {require: false});
if (!model) {
return null;
}
return this.#modelToMention(model);
}
/**
* @param {import('@tryghost/webmentions/lib/Mention')} mention
* @returns {Promise<void>}
*/
async save(mention) {
const data = {
id: mention.id.toHexString(),
source: mention.source.href,
source_title: mention.sourceTitle,
source_site_title: mention.sourceSiteTitle,
source_excerpt: mention.sourceExcerpt,
source_author: mention.sourceAuthor,
source_featured_image: mention.sourceFeaturedImage?.href,
source_favicon: mention.sourceFavicon?.href,
target: mention.target.href,
resource_id: mention.resourceId?.toHexString(),
resource_type: mention.resourceId ? 'post' : null,
payload: mention.payload ? JSON.stringify(mention.payload) : null
};
const existing = await this.#MentionModel.findOne({id: data.id}, {require: false});
if (!existing) {
await this.#MentionModel.add(data);
} else {
await this.#MentionModel.edit(data, {
id: data.id
});
}
}
};

View File

@ -12,8 +12,8 @@ module.exports = class WebmentionMetadata {
title: data.metadata.title, title: data.metadata.title,
excerpt: data.metadata.description, excerpt: data.metadata.description,
author: data.metadata.author, author: data.metadata.author,
image: new URL(data.metadata.thumbnail), image: data.metadata.thumbnail ? new URL(data.metadata.thumbnail) : null,
favicon: new URL(data.metadata.icon) favicon: data.metadata.icon ? new URL(data.metadata.icon) : null
}; };
return result; return result;
} }

View File

@ -2,11 +2,12 @@ const ObjectID = require('bson-objectid').default;
const MentionController = require('./MentionController'); const MentionController = require('./MentionController');
const WebmentionMetadata = require('./WebmentionMetadata'); const WebmentionMetadata = require('./WebmentionMetadata');
const { const {
InMemoryMentionRepository,
MentionsAPI, MentionsAPI,
MentionSendingService, MentionSendingService,
MentionDiscoveryService MentionDiscoveryService
} = require('@tryghost/webmentions'); } = require('@tryghost/webmentions');
const BookshelfMentionRepository = require('./BookshelfMentionRepository');
const models = require('../../models');
const events = require('../../lib/common/events'); const events = require('../../lib/common/events');
const externalRequest = require('../../../server/lib/request-external.js'); const externalRequest = require('../../../server/lib/request-external.js');
const urlUtils = require('../../../shared/url-utils'); const urlUtils = require('../../../shared/url-utils');
@ -22,7 +23,9 @@ function getPostUrl(post) {
module.exports = { module.exports = {
controller: new MentionController(), controller: new MentionController(),
async init() { async init() {
const repository = new InMemoryMentionRepository(); const repository = new BookshelfMentionRepository({
MentionModel: models.Mention
});
const webmentionMetadata = new WebmentionMetadata(); const webmentionMetadata = new WebmentionMetadata();
const discoveryService = new MentionDiscoveryService({externalRequest}); const discoveryService = new MentionDiscoveryService({externalRequest});
const api = new MentionsAPI({ const api = new MentionsAPI({
@ -62,24 +65,6 @@ module.exports = {
this.controller.init({api}); this.controller.init({api});
const addMocks = () => {
if (!urlService.hasFinished()) {
setTimeout(addMocks, 100);
return;
}
this.controller.receive({
data: {
source: 'https://brid.gy/repost/twitter/KiaKamgar/1615735511137624064/1615738476875366401',
target: 'https://valid-url-for-your.site'
}
});
return;
};
addMocks();
const sendingService = new MentionSendingService({ const sendingService = new MentionSendingService({
discoveryService, discoveryService,
externalRequest, externalRequest,

View File

@ -0,0 +1,81 @@
const {agentProvider, fixtureManager, mockManager, matchers, sleep} = require('../../utils/e2e-framework');
const models = require('../../../core/server/models');
const assert = require('assert');
const urlUtils = require('../../../core/shared/url-utils');
const nock = require('nock');
describe('Webmentions (receiving)', function () {
let agent;
before(async function () {
agent = await agentProvider.getWebmentionsAPIAgent();
await fixtureManager.init('posts');
nock.disableNetConnect();
});
after(function () {
nock.cleanAll();
nock.enableNetConnect();
});
it('can receive a webmention', async function () {
const url = new URL('http://testpage.com/external-article/');
const html = `
<html><head><title>Test Page</title><meta name="description" content="Test description"><meta name="author" content="John Doe"></head><body></body></html>
`;
nock(url.href)
.get('/')
.reply(200, html, {'content-type': 'text/html'});
await agent.post('/receive')
.body({
source: 'http://testpage.com/external-article/',
target: urlUtils.getSiteUrl() + 'integrations/',
withExtension: true // test payload recorded
})
.expectStatus(202);
// todo: remove sleep in future
await sleep(2000);
const mention = await models.Mention.findOne({source: 'http://testpage.com/external-article/'});
assert(mention);
assert.equal(mention.get('target'), urlUtils.getSiteUrl() + 'integrations/');
assert.ok(mention.get('resource_id'));
assert.equal(mention.get('resource_type'), 'post');
assert.equal(mention.get('source_title'), 'Test Page');
assert.equal(mention.get('source_excerpt'), 'Test description');
assert.equal(mention.get('source_author'), 'John Doe');
assert.equal(mention.get('payload'), JSON.stringify({
withExtension: true
}));
});
it('can receive a webmention to homepage', async function () {
const url = new URL('http://testpage.com/external-article-2/');
const html = `
<html><head><title>Test Page</title><meta name="description" content="Test description"><meta name="author" content="John Doe"></head><body></body></html>
`;
nock(url.href)
.get('/')
.reply(200, html, {'content-type': 'text/html'});
await agent.post('/receive')
.body({
source: 'http://testpage.com/external-article-2/',
target: urlUtils.getSiteUrl()
})
.expectStatus(202);
// todo: remove sleep in future
await sleep(2000);
const mention = await models.Mention.findOne({source: 'http://testpage.com/external-article-2/'});
assert(mention);
assert.equal(mention.get('target'), urlUtils.getSiteUrl());
assert.ok(!mention.get('resource_id'));
assert.equal(mention.get('resource_type'), null);
assert.equal(mention.get('source_title'), 'Test Page');
assert.equal(mention.get('source_excerpt'), 'Test description');
assert.equal(mention.get('source_author'), 'John Doe');
assert.equal(mention.get('payload'), JSON.stringify({}));
});
});

View File

@ -231,6 +231,31 @@ const getMembersAPIAgent = async () => {
} }
}; };
/**
* Creates a MembersAPITestAgent which is a drop-in substitution for supertest
* It is automatically hooked up to the Members API so you can make requests to e.g.
* agent.get('/webhooks/stripe/') without having to worry about URL paths
*
* @returns {Promise<InstanceType<GhostAPITestAgent>>} agent
*/
const getWebmentionsAPIAgent = async () => {
const bootOptions = {
frontend: true
};
try {
const app = await startGhost(bootOptions);
const originURL = configUtils.config.get('url');
return new GhostAPITestAgent(app, {
apiURL: '/webmentions/',
originURL
});
} catch (error) {
error.message = `Unable to create test agent. ${error.message}`;
throw error;
}
};
/** /**
* Creates a GhostAPITestAgent, which is a drop-in substitution for supertest * Creates a GhostAPITestAgent, which is a drop-in substitution for supertest
* It is automatically hooked up to the Ghost API so you can make requests to e.g. * It is automatically hooked up to the Ghost API so you can make requests to e.g.
@ -381,6 +406,7 @@ module.exports = {
agentProvider: { agentProvider: {
getAdminAPIAgent, getAdminAPIAgent,
getMembersAPIAgent, getMembersAPIAgent,
getWebmentionsAPIAgent,
getContentAPIAgent, getContentAPIAgent,
getAgentsForMembers, getAgentsForMembers,
getGhostAPIAgent, getGhostAPIAgent,

View File

@ -21,6 +21,11 @@ describe('MentionSendingService', function () {
sinon.restore(); sinon.restore();
}); });
after(function () {
nock.cleanAll();
nock.enableNetConnect();
});
describe('listen', function () { describe('listen', function () {
it('Calls on post.edited', async function () { it('Calls on post.edited', async function () {
const service = new MentionSendingService({}); const service = new MentionSendingService({});