Added BookshelfMentionRepository implementation (#16156)

fixes https://github.com/TryGhost/Team/issues/2418

This stores the received webmentions in the database.

Co-authored-by: Simon Backx <simon@ghost.org>

ghost/core/core/server/api/endpoints/mentions.js

@@ -25,8 +25,9 @@ module.exports = {
         response: {
             format: 'plain'
         },
-        query(frame) {
-            return mentions.controller.receive(frame);
+        async query(frame) {
+            await mentions.controller.receive(frame);
+            return null;
         }
     }
 };

ghost/core/core/server/models/mention.js

@@ -0,0 +1,9 @@
+const ghostBookshelf = require('./base');
+
+const Mention = ghostBookshelf.Model.extend({
+    tableName: 'mentions'
+});
+
+module.exports = {
+    Mention: ghostBookshelf.model('Mention', Mention)
+};

ghost/core/core/server/services/mentions/BookshelfMentionRepository.js

@@ -0,0 +1,117 @@
+const {Mention} = require('@tryghost/webmentions');
+const logging = require('@tryghost/logging');
+
+/**
+ * @typedef {import('@tryghost/webmentions/lib/MentionsAPI').IMentionRepository} IMentionRepository
+ */
+
+/**
+ * @template Model
+ * @typedef {import('@tryghost/webmentions/lib/MentionsAPI').Page<Model>} Page
+ */
+
+/**
+ * @typedef {import('@tryghost/webmentions/lib/MentionsAPI').GetPageOptions} GetPageOptions
+ */
+
+/**
+ * @implements {IMentionRepository}
+ */
+module.exports = class BookshelfMentionRepository {
+    /** @type {Object} */
+    #MentionModel;
+
+    /**
+     * @param {object} deps
+     * @param {object} deps.MentionModel Bookshelf Model
+     */
+    constructor(deps) {
+        this.#MentionModel = deps.MentionModel;
+    }
+
+    #modelToMention(model) {
+        let payload;
+        try {
+            payload = JSON.parse(model.get('payload'));
+        } catch (err) {
+            logging.error(err);
+            payload = {};
+        }
+        return Mention.create({
+            id: model.get('id'),
+            source: model.get('source'),
+            target: model.get('target'),
+            timestamp: model.get('created_at'),
+            payload,
+            resourceId: model.get('resource_id'),
+            sourceTitle: model.get('source_title'),
+            sourceSiteTitle: model.get('source_site_title'),
+            sourceAuthor: model.get('source_author'),
+            sourceExcerpt: model.get('source_excerpt'),
+            sourceFavicon: model.get('source_favicon'),
+            sourceFeaturedImaged: model.get('source_featured_image')
+        });
+    }
+
+    /**
+     * @param {GetPageOptions} options
+     * @returns {Promise<Page<import('@tryghost/webmentions/lib/Mention')>>}
+     */
+    async getPage(options) {
+        const page = await this.#MentionModel.findPage(options);
+
+        return {
+            data: await Promise.all(page.data.map(model => this.#modelToMention(model))),
+            meta: page.meta
+        };
+    }
+
+    /**
+     * @param {URL} source
+     * @param {URL} target
+     * @returns {Promise<import('@tryghost/webmentions/lib/Mention')|null>}
+     */
+    async getBySourceAndTarget(source, target) {
+        const model = await this.#MentionModel.findOne({
+            source: source.href,
+            target: target.href
+        }, {require: false});
+
+        if (!model) {
+            return null;
+        }
+
+        return this.#modelToMention(model);
+    }
+
+    /**
+     * @param {import('@tryghost/webmentions/lib/Mention')} mention
+     * @returns {Promise<void>}
+     */
+    async save(mention) {
+        const data = {
+            id: mention.id.toHexString(),
+            source: mention.source.href,
+            source_title: mention.sourceTitle,
+            source_site_title: mention.sourceSiteTitle,
+            source_excerpt: mention.sourceExcerpt,
+            source_author: mention.sourceAuthor,
+            source_featured_image: mention.sourceFeaturedImage?.href,
+            source_favicon: mention.sourceFavicon?.href,
+            target: mention.target.href,
+            resource_id: mention.resourceId?.toHexString(),
+            resource_type: mention.resourceId ? 'post' : null,
+            payload: mention.payload ? JSON.stringify(mention.payload) : null
+        };
+
+        const existing = await this.#MentionModel.findOne({id: data.id}, {require: false});
+
+        if (!existing) {
+            await this.#MentionModel.add(data);
+        } else {
+            await this.#MentionModel.edit(data, {
+                id: data.id
+            });
+        }
+    }
+};

ghost/core/core/server/services/mentions/WebmentionMetadata.js

@@ -12,8 +12,8 @@ module.exports = class WebmentionMetadata {
             title: data.metadata.title,
             excerpt: data.metadata.description,
             author: data.metadata.author,
-            image: new URL(data.metadata.thumbnail),
-            favicon: new URL(data.metadata.icon)
+            image: data.metadata.thumbnail ? new URL(data.metadata.thumbnail) : null,
+            favicon: data.metadata.icon ? new URL(data.metadata.icon) : null
         };
         return result;
     }

ghost/core/core/server/services/mentions/service.js

@@ -2,11 +2,12 @@ const ObjectID = require('bson-objectid').default;
 const MentionController = require('./MentionController');
 const WebmentionMetadata = require('./WebmentionMetadata');
 const {
-    InMemoryMentionRepository,
     MentionsAPI,
     MentionSendingService,
     MentionDiscoveryService
 } = require('@tryghost/webmentions');
+const BookshelfMentionRepository = require('./BookshelfMentionRepository');
+const models = require('../../models');
 const events = require('../../lib/common/events');
 const externalRequest = require('../../../server/lib/request-external.js');
 const urlUtils = require('../../../shared/url-utils');
@@ -22,7 +23,9 @@ function getPostUrl(post) {
 module.exports = {
     controller: new MentionController(),
     async init() {
-        const repository = new InMemoryMentionRepository();
+        const repository = new BookshelfMentionRepository({
+            MentionModel: models.Mention
+        });
         const webmentionMetadata = new WebmentionMetadata();
         const discoveryService = new MentionDiscoveryService({externalRequest});
         const api = new MentionsAPI({
@@ -62,24 +65,6 @@ module.exports = {
 
         this.controller.init({api});
 
-        const addMocks = () => {
-            if (!urlService.hasFinished()) {
-                setTimeout(addMocks, 100);
-                return;
-            }
-
-            this.controller.receive({
-                data: {
-                    source: 'https://brid.gy/repost/twitter/KiaKamgar/1615735511137624064/1615738476875366401',
-                    target: 'https://valid-url-for-your.site'
-                }
-            });
-
-            return;
-        };
-
-        addMocks();
-
         const sendingService = new MentionSendingService({
             discoveryService,
             externalRequest,

ghost/core/test/e2e-api/webmentions/webmentions.test.js

@@ -0,0 +1,81 @@
+const {agentProvider, fixtureManager, mockManager, matchers, sleep} = require('../../utils/e2e-framework');
+const models = require('../../../core/server/models');
+const assert = require('assert');
+const urlUtils = require('../../../core/shared/url-utils');
+const nock = require('nock');
+
+describe('Webmentions (receiving)', function () {
+    let agent;
+    before(async function () {
+        agent = await agentProvider.getWebmentionsAPIAgent();
+        await fixtureManager.init('posts');
+        nock.disableNetConnect();
+    });
+
+    after(function () {
+        nock.cleanAll();
+        nock.enableNetConnect();
+    });
+
+    it('can receive a webmention', async function () {
+        const url = new URL('http://testpage.com/external-article/');
+        const html = `
+                <html><head><title>Test Page</title><meta name="description" content="Test description"><meta name="author" content="John Doe"></head><body></body></html>
+            `;
+        nock(url.href)
+            .get('/')
+            .reply(200, html, {'content-type': 'text/html'});
+
+        await agent.post('/receive')
+            .body({
+                source: 'http://testpage.com/external-article/',
+                target: urlUtils.getSiteUrl() + 'integrations/',
+                withExtension: true // test payload recorded
+            })
+            .expectStatus(202);
+
+        // todo: remove sleep in future
+        await sleep(2000);
+
+        const mention = await models.Mention.findOne({source: 'http://testpage.com/external-article/'});
+        assert(mention);
+        assert.equal(mention.get('target'), urlUtils.getSiteUrl() + 'integrations/');
+        assert.ok(mention.get('resource_id'));
+        assert.equal(mention.get('resource_type'), 'post');
+        assert.equal(mention.get('source_title'), 'Test Page');
+        assert.equal(mention.get('source_excerpt'), 'Test description');
+        assert.equal(mention.get('source_author'), 'John Doe');
+        assert.equal(mention.get('payload'), JSON.stringify({
+            withExtension: true
+        }));
+    });
+    it('can receive a webmention to homepage', async function () {
+        const url = new URL('http://testpage.com/external-article-2/');
+        const html = `
+                <html><head><title>Test Page</title><meta name="description" content="Test description"><meta name="author" content="John Doe"></head><body></body></html>
+            `;
+        nock(url.href)
+            .get('/')
+            .reply(200, html, {'content-type': 'text/html'});
+
+        await agent.post('/receive')
+            .body({
+                source: 'http://testpage.com/external-article-2/',
+                target: urlUtils.getSiteUrl()
+            })
+            .expectStatus(202);
+
+        // todo: remove sleep in future
+        await sleep(2000);
+
+        const mention = await models.Mention.findOne({source: 'http://testpage.com/external-article-2/'});
+        assert(mention);
+        assert.equal(mention.get('target'), urlUtils.getSiteUrl());
+        assert.ok(!mention.get('resource_id'));
+        assert.equal(mention.get('resource_type'), null);
+        assert.equal(mention.get('source_title'), 'Test Page');
+        assert.equal(mention.get('source_excerpt'), 'Test description');
+        assert.equal(mention.get('source_author'), 'John Doe');
+        assert.equal(mention.get('payload'), JSON.stringify({}));
+    });
+});

ghost/core/test/utils/e2e-framework.js

@@ -231,6 +231,31 @@ const getMembersAPIAgent = async () => {
     }
 };
 
+/**
+ * Creates a MembersAPITestAgent which is a drop-in substitution for supertest
+ * It is automatically hooked up to the Members API so you can make requests to e.g.
+ * agent.get('/webhooks/stripe/') without having to worry about URL paths
+ *
+ * @returns {Promise<InstanceType<GhostAPITestAgent>>} agent
+ */
+const getWebmentionsAPIAgent = async () => {
+    const bootOptions = {
+        frontend: true
+    };
+    try {
+        const app = await startGhost(bootOptions);
+        const originURL = configUtils.config.get('url');
+
+        return new GhostAPITestAgent(app, {
+            apiURL: '/webmentions/',
+            originURL
+        });
+    } catch (error) {
+        error.message = `Unable to create test agent. ${error.message}`;
+        throw error;
+    }
+};
+
 /**
  * Creates a GhostAPITestAgent, which is a drop-in substitution for supertest
  * It is automatically hooked up to the Ghost API so you can make requests to e.g.
@@ -381,6 +406,7 @@ module.exports = {
     agentProvider: {
         getAdminAPIAgent,
         getMembersAPIAgent,
+        getWebmentionsAPIAgent,
         getContentAPIAgent,
         getAgentsForMembers,
         getGhostAPIAgent,

ghost/oembed-service/lib/oembed-service.js

@@ -116,7 +116,7 @@ class OEmbed {
     /**
      * @param {string} url
      * @param {Object} options
-     * 
+     *
      * @returns {Promise<{url: string, body: any, headers: any}>}
      */
     async fetchPage(url, options) {
@@ -134,7 +134,7 @@ class OEmbed {
 
     /**
      * @param {string} url
-     * 
+     *
      * @returns {Promise<{url: string, body: string}>}
      */
     async fetchPageHtml(url) {
@@ -180,7 +180,7 @@ class OEmbed {
 
     /**
      * @param {string} url
-     * 
+     *
      * @returns {Promise<{url: string, body: Object}>}
      */
     async fetchPageJson(url) {
@@ -195,11 +195,11 @@ class OEmbed {
             url: pageUrl
         };
     }
-    
+
     /**
      * @param {string} url
      * @param {string} html
-     * 
+     *
      * @returns {Promise<Object>}
      */
     async fetchBookmarkData(url, html) {
@@ -215,7 +215,7 @@ class OEmbed {
         ]);
 
         let scraperResponse;
-        
+
         try {
             scraperResponse = await metascraper({html, url});
         } catch (err) {
@@ -383,7 +383,7 @@ class OEmbed {
 
             // attempt to fetch oembed
 
-            // In case response was a redirect, see if we were 
+            // In case response was a redirect, see if we were
             // redirected to a known oembed
             if (pageUrl !== url) {
                 const {url: providerUrl, provider} = findUrlWithProvider(pageUrl);

ghost/webmentions/test/MentionSendingService.test.js

@@ -21,6 +21,11 @@ describe('MentionSendingService', function () {
         sinon.restore();
     });
 
+    after(function () {
+        nock.cleanAll();
+        nock.enableNetConnect();
+    });
+
     describe('listen', function () {
         it('Calls on post.edited', async function () {
             const service = new MentionSendingService({});