diff --git a/ghost/core/core/server/api/endpoints/mentions.js b/ghost/core/core/server/api/endpoints/mentions.js index 720c80d44e..8c204a6e66 100644 --- a/ghost/core/core/server/api/endpoints/mentions.js +++ b/ghost/core/core/server/api/endpoints/mentions.js @@ -25,8 +25,9 @@ module.exports = { response: { format: 'plain' }, - query(frame) { - return mentions.controller.receive(frame); + async query(frame) { + await mentions.controller.receive(frame); + return null; } } }; diff --git a/ghost/core/core/server/models/mention.js b/ghost/core/core/server/models/mention.js new file mode 100644 index 0000000000..32901dd65d --- /dev/null +++ b/ghost/core/core/server/models/mention.js @@ -0,0 +1,9 @@ +const ghostBookshelf = require('./base'); + +const Mention = ghostBookshelf.Model.extend({ + tableName: 'mentions' +}); + +module.exports = { + Mention: ghostBookshelf.model('Mention', Mention) +}; diff --git a/ghost/core/core/server/services/mentions/BookshelfMentionRepository.js b/ghost/core/core/server/services/mentions/BookshelfMentionRepository.js new file mode 100644 index 0000000000..511f38028c --- /dev/null +++ b/ghost/core/core/server/services/mentions/BookshelfMentionRepository.js @@ -0,0 +1,117 @@ +const {Mention} = require('@tryghost/webmentions'); +const logging = require('@tryghost/logging'); + +/** + * @typedef {import('@tryghost/webmentions/lib/MentionsAPI').IMentionRepository} IMentionRepository + */ + +/** + * @template Model + * @typedef {import('@tryghost/webmentions/lib/MentionsAPI').Page} Page + */ + +/** + * @typedef {import('@tryghost/webmentions/lib/MentionsAPI').GetPageOptions} GetPageOptions + */ + +/** + * @implements {IMentionRepository} + */ +module.exports = class BookshelfMentionRepository { + /** @type {Object} */ + #MentionModel; + + /** + * @param {object} deps + * @param {object} deps.MentionModel Bookshelf Model + */ + constructor(deps) { + this.#MentionModel = deps.MentionModel; + } + + #modelToMention(model) { + let payload; + try { + payload = JSON.parse(model.get('payload')); + } catch (err) { + logging.error(err); + payload = {}; + } + return Mention.create({ + id: model.get('id'), + source: model.get('source'), + target: model.get('target'), + timestamp: model.get('created_at'), + payload, + resourceId: model.get('resource_id'), + sourceTitle: model.get('source_title'), + sourceSiteTitle: model.get('source_site_title'), + sourceAuthor: model.get('source_author'), + sourceExcerpt: model.get('source_excerpt'), + sourceFavicon: model.get('source_favicon'), + sourceFeaturedImaged: model.get('source_featured_image') + }); + } + + /** + * @param {GetPageOptions} options + * @returns {Promise>} + */ + async getPage(options) { + const page = await this.#MentionModel.findPage(options); + + return { + data: await Promise.all(page.data.map(model => this.#modelToMention(model))), + meta: page.meta + }; + } + + /** + * @param {URL} source + * @param {URL} target + * @returns {Promise} + */ + async getBySourceAndTarget(source, target) { + const model = await this.#MentionModel.findOne({ + source: source.href, + target: target.href + }, {require: false}); + + if (!model) { + return null; + } + + return this.#modelToMention(model); + } + + /** + * @param {import('@tryghost/webmentions/lib/Mention')} mention + * @returns {Promise} + */ + async save(mention) { + const data = { + id: mention.id.toHexString(), + source: mention.source.href, + source_title: mention.sourceTitle, + source_site_title: mention.sourceSiteTitle, + source_excerpt: mention.sourceExcerpt, + source_author: mention.sourceAuthor, + source_featured_image: mention.sourceFeaturedImage?.href, + source_favicon: mention.sourceFavicon?.href, + target: mention.target.href, + resource_id: mention.resourceId?.toHexString(), + resource_type: mention.resourceId ? 'post' : null, + payload: mention.payload ? JSON.stringify(mention.payload) : null + }; + + const existing = await this.#MentionModel.findOne({id: data.id}, {require: false}); + + if (!existing) { + await this.#MentionModel.add(data); + } else { + await this.#MentionModel.edit(data, { + id: data.id + }); + } + } +}; diff --git a/ghost/core/core/server/services/mentions/WebmentionMetadata.js b/ghost/core/core/server/services/mentions/WebmentionMetadata.js index 2668a022e0..cd61ea5ddb 100644 --- a/ghost/core/core/server/services/mentions/WebmentionMetadata.js +++ b/ghost/core/core/server/services/mentions/WebmentionMetadata.js @@ -12,8 +12,8 @@ module.exports = class WebmentionMetadata { title: data.metadata.title, excerpt: data.metadata.description, author: data.metadata.author, - image: new URL(data.metadata.thumbnail), - favicon: new URL(data.metadata.icon) + image: data.metadata.thumbnail ? new URL(data.metadata.thumbnail) : null, + favicon: data.metadata.icon ? new URL(data.metadata.icon) : null }; return result; } diff --git a/ghost/core/core/server/services/mentions/service.js b/ghost/core/core/server/services/mentions/service.js index 3afce9118c..6820cf2792 100644 --- a/ghost/core/core/server/services/mentions/service.js +++ b/ghost/core/core/server/services/mentions/service.js @@ -2,11 +2,12 @@ const ObjectID = require('bson-objectid').default; const MentionController = require('./MentionController'); const WebmentionMetadata = require('./WebmentionMetadata'); const { - InMemoryMentionRepository, MentionsAPI, MentionSendingService, MentionDiscoveryService } = require('@tryghost/webmentions'); +const BookshelfMentionRepository = require('./BookshelfMentionRepository'); +const models = require('../../models'); const events = require('../../lib/common/events'); const externalRequest = require('../../../server/lib/request-external.js'); const urlUtils = require('../../../shared/url-utils'); @@ -22,7 +23,9 @@ function getPostUrl(post) { module.exports = { controller: new MentionController(), async init() { - const repository = new InMemoryMentionRepository(); + const repository = new BookshelfMentionRepository({ + MentionModel: models.Mention + }); const webmentionMetadata = new WebmentionMetadata(); const discoveryService = new MentionDiscoveryService({externalRequest}); const api = new MentionsAPI({ @@ -62,24 +65,6 @@ module.exports = { this.controller.init({api}); - const addMocks = () => { - if (!urlService.hasFinished()) { - setTimeout(addMocks, 100); - return; - } - - this.controller.receive({ - data: { - source: 'https://brid.gy/repost/twitter/KiaKamgar/1615735511137624064/1615738476875366401', - target: 'https://valid-url-for-your.site' - } - }); - - return; - }; - - addMocks(); - const sendingService = new MentionSendingService({ discoveryService, externalRequest, diff --git a/ghost/core/test/e2e-api/webmentions/webmentions.test.js b/ghost/core/test/e2e-api/webmentions/webmentions.test.js new file mode 100644 index 0000000000..43ec1e933a --- /dev/null +++ b/ghost/core/test/e2e-api/webmentions/webmentions.test.js @@ -0,0 +1,81 @@ +const {agentProvider, fixtureManager, mockManager, matchers, sleep} = require('../../utils/e2e-framework'); +const models = require('../../../core/server/models'); +const assert = require('assert'); +const urlUtils = require('../../../core/shared/url-utils'); +const nock = require('nock'); + +describe('Webmentions (receiving)', function () { + let agent; + before(async function () { + agent = await agentProvider.getWebmentionsAPIAgent(); + await fixtureManager.init('posts'); + nock.disableNetConnect(); + }); + + after(function () { + nock.cleanAll(); + nock.enableNetConnect(); + }); + + it('can receive a webmention', async function () { + const url = new URL('http://testpage.com/external-article/'); + const html = ` + Test Page + `; + nock(url.href) + .get('/') + .reply(200, html, {'content-type': 'text/html'}); + + await agent.post('/receive') + .body({ + source: 'http://testpage.com/external-article/', + target: urlUtils.getSiteUrl() + 'integrations/', + withExtension: true // test payload recorded + }) + .expectStatus(202); + + // todo: remove sleep in future + await sleep(2000); + + const mention = await models.Mention.findOne({source: 'http://testpage.com/external-article/'}); + assert(mention); + assert.equal(mention.get('target'), urlUtils.getSiteUrl() + 'integrations/'); + assert.ok(mention.get('resource_id')); + assert.equal(mention.get('resource_type'), 'post'); + assert.equal(mention.get('source_title'), 'Test Page'); + assert.equal(mention.get('source_excerpt'), 'Test description'); + assert.equal(mention.get('source_author'), 'John Doe'); + assert.equal(mention.get('payload'), JSON.stringify({ + withExtension: true + })); + }); + it('can receive a webmention to homepage', async function () { + const url = new URL('http://testpage.com/external-article-2/'); + const html = ` + Test Page + `; + nock(url.href) + .get('/') + .reply(200, html, {'content-type': 'text/html'}); + + await agent.post('/receive') + .body({ + source: 'http://testpage.com/external-article-2/', + target: urlUtils.getSiteUrl() + }) + .expectStatus(202); + + // todo: remove sleep in future + await sleep(2000); + + const mention = await models.Mention.findOne({source: 'http://testpage.com/external-article-2/'}); + assert(mention); + assert.equal(mention.get('target'), urlUtils.getSiteUrl()); + assert.ok(!mention.get('resource_id')); + assert.equal(mention.get('resource_type'), null); + assert.equal(mention.get('source_title'), 'Test Page'); + assert.equal(mention.get('source_excerpt'), 'Test description'); + assert.equal(mention.get('source_author'), 'John Doe'); + assert.equal(mention.get('payload'), JSON.stringify({})); + }); +}); diff --git a/ghost/core/test/utils/e2e-framework.js b/ghost/core/test/utils/e2e-framework.js index 28717d8041..307874deb9 100644 --- a/ghost/core/test/utils/e2e-framework.js +++ b/ghost/core/test/utils/e2e-framework.js @@ -231,6 +231,31 @@ const getMembersAPIAgent = async () => { } }; +/** + * Creates a MembersAPITestAgent which is a drop-in substitution for supertest + * It is automatically hooked up to the Members API so you can make requests to e.g. + * agent.get('/webhooks/stripe/') without having to worry about URL paths + * + * @returns {Promise>} agent + */ +const getWebmentionsAPIAgent = async () => { + const bootOptions = { + frontend: true + }; + try { + const app = await startGhost(bootOptions); + const originURL = configUtils.config.get('url'); + + return new GhostAPITestAgent(app, { + apiURL: '/webmentions/', + originURL + }); + } catch (error) { + error.message = `Unable to create test agent. ${error.message}`; + throw error; + } +}; + /** * Creates a GhostAPITestAgent, which is a drop-in substitution for supertest * It is automatically hooked up to the Ghost API so you can make requests to e.g. @@ -381,6 +406,7 @@ module.exports = { agentProvider: { getAdminAPIAgent, getMembersAPIAgent, + getWebmentionsAPIAgent, getContentAPIAgent, getAgentsForMembers, getGhostAPIAgent, diff --git a/ghost/oembed-service/lib/oembed-service.js b/ghost/oembed-service/lib/oembed-service.js index 77fe8ba181..1bd7dbd7fe 100644 --- a/ghost/oembed-service/lib/oembed-service.js +++ b/ghost/oembed-service/lib/oembed-service.js @@ -116,7 +116,7 @@ class OEmbed { /** * @param {string} url * @param {Object} options - * + * * @returns {Promise<{url: string, body: any, headers: any}>} */ async fetchPage(url, options) { @@ -134,7 +134,7 @@ class OEmbed { /** * @param {string} url - * + * * @returns {Promise<{url: string, body: string}>} */ async fetchPageHtml(url) { @@ -180,7 +180,7 @@ class OEmbed { /** * @param {string} url - * + * * @returns {Promise<{url: string, body: Object}>} */ async fetchPageJson(url) { @@ -195,11 +195,11 @@ class OEmbed { url: pageUrl }; } - + /** * @param {string} url * @param {string} html - * + * * @returns {Promise} */ async fetchBookmarkData(url, html) { @@ -215,7 +215,7 @@ class OEmbed { ]); let scraperResponse; - + try { scraperResponse = await metascraper({html, url}); } catch (err) { @@ -383,7 +383,7 @@ class OEmbed { // attempt to fetch oembed - // In case response was a redirect, see if we were + // In case response was a redirect, see if we were // redirected to a known oembed if (pageUrl !== url) { const {url: providerUrl, provider} = findUrlWithProvider(pageUrl); diff --git a/ghost/webmentions/test/MentionSendingService.test.js b/ghost/webmentions/test/MentionSendingService.test.js index afb84bdec2..761852741a 100644 --- a/ghost/webmentions/test/MentionSendingService.test.js +++ b/ghost/webmentions/test/MentionSendingService.test.js @@ -21,6 +21,11 @@ describe('MentionSendingService', function () { sinon.restore(); }); + after(function () { + nock.cleanAll(); + nock.enableNetConnect(); + }); + describe('listen', function () { it('Calls on post.edited', async function () { const service = new MentionSendingService({});