Added well known recommendations service (#17895)

fixes https://github.com/TryGhost/Product/issues/3797 
fixes https://github.com/TryGhost/Product/issues/3776 
fixes https://github.com/TryGhost/Product/issues/3798

- Added support for storing json webmentions
- Improved handling deleted webmentions (set deleted to true instead of verified to false)
This commit is contained in:
Simon Backx 2023-08-31 16:57:18 +02:00 committed by GitHub
parent e97b71dc52
commit 96fefaea69
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 338 additions and 77 deletions

View File

@ -26,7 +26,7 @@ function matchCacheKey(req, cache) {
return true;
}
function createPublicFileMiddleware(location, file, mime, maxAge) {
function createPublicFileMiddleware(location, file, mime, maxAge, options = {}) {
let cache;
// These files are provided by Ghost, and therefore live inside of the core folder
const staticFilePath = config.get('paths').publicFilePath;
@ -45,7 +45,7 @@ function createPublicFileMiddleware(location, file, mime, maxAge) {
}
// send image files directly and let express handle content-length, etag, etc
if (mime.match(/^image/)) {
if (mime.match(/^image/) || options.disableServerCache) {
return res.sendFile(filePath, (err) => {
if (err && err.status === 404) {
// ensure we're triggering basic asset 404 and not a templated 404
@ -101,8 +101,8 @@ function createPublicFileMiddleware(location, file, mime, maxAge) {
// ### servePublicFile Middleware
// Handles requests to robots.txt and favicon.ico (and caches them)
function servePublicFile(location, file, type, maxAge) {
const publicFileMiddleware = createPublicFileMiddleware(location, file, type, maxAge);
function servePublicFile(location, file, type, maxAge, options = {}) {
const publicFileMiddleware = createPublicFileMiddleware(location, file, type, maxAge, options);
return function servePublicFileMiddleware(req, res, next) {
if (req.path === '/' + file) {

View File

@ -98,6 +98,9 @@ module.exports = function setupSiteApp(routerConfig) {
(req, res, next) => membersService.api.middleware.wellKnown(req, res, next)
);
// Recommendations well-known
siteApp.use(mw.servePublicFile('built', '.well-known/recommendations.json', 'application/json', config.get('caching:publicAssets:maxAge'), {disableServerCache: true}));
// setup middleware for internal apps
// @TODO: refactor this to be a proper app middleware hook for internal apps
config.get('apps:internal').forEach((appName) => {

View File

@ -14,7 +14,8 @@ module.exports = class WebmentionMetadata {
author: data.metadata.author,
image: data.metadata.thumbnail ? new URL(data.metadata.thumbnail) : null,
favicon: data.metadata.icon ? new URL(data.metadata.icon) : null,
body: data.body
body: data.body,
contentType: data.contentType
};
return result;
}

View File

@ -28,6 +28,8 @@ module.exports = {
/** @type {import('@tryghost/webmentions/lib/MentionsAPI')} */
api: null,
controller: new MentionController(),
/** @type {import('@tryghost/webmentions/lib/MentionSendingService')} */
sendingService: null,
didInit: false,
async init() {
if (this.didInit) {
@ -107,5 +109,7 @@ module.exports = {
}
});
sendingService.listen(events);
this.sendingService = sendingService;
}
};

View File

@ -19,15 +19,34 @@ class RecommendationServiceWrapper {
return;
}
const {InMemoryRecommendationRepository, RecommendationService, RecommendationController} = require('@tryghost/recommendations');
const config = require('../../../shared/config');
const urlUtils = require('../../../shared/url-utils');
const {InMemoryRecommendationRepository, RecommendationService, RecommendationController, WellknownService} = require('@tryghost/recommendations');
const mentions = require('../mentions');
if (!mentions.sendingService) {
// eslint-disable-next-line ghost/ghost-custom/no-native-error
throw new Error('MentionSendingService not intialized, but this is a dependency of RecommendationServiceWrapper. Check boot order.');
}
const wellknownService = new WellknownService({
dir: config.getContentPath('public'),
urlUtils
});
this.repository = new InMemoryRecommendationRepository();
this.service = new RecommendationService({
repository: this.repository
repository: this.repository,
wellknownService,
mentionSendingService: mentions.sendingService
});
this.controller = new RecommendationController({
service: this.service
});
// eslint-disable-next-line no-console
this.service.init().catch(console.error);
}
}

View File

@ -13,7 +13,7 @@ Object {
"reason": "Because dogs are cute",
"title": "Dog Pictures",
"updated_at": null,
"url": "https://dogpictures.com",
"url": "https://dogpictures.com/",
},
],
}
@ -23,7 +23,7 @@ exports[`Recommendations Admin API Can add a full recommendation 2: [headers] 1`
Object {
"access-control-allow-origin": "http://127.0.0.1:2369",
"cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
"content-length": "353",
"content-length": "354",
"content-type": "application/json; charset=utf-8",
"content-version": StringMatching /v\\\\d\\+\\\\\\.\\\\d\\+/,
"etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
@ -47,7 +47,7 @@ Object {
"reason": null,
"title": "Dog Pictures",
"updated_at": null,
"url": "https://dogpictures.com",
"url": "https://dogpictures.com/",
},
],
}
@ -57,7 +57,7 @@ exports[`Recommendations Admin API Can add a minimal recommendation 2: [headers]
Object {
"access-control-allow-origin": "http://127.0.0.1:2369",
"cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
"content-length": "262",
"content-length": "263",
"content-type": "application/json; charset=utf-8",
"content-version": StringMatching /v\\\\d\\+\\\\\\.\\\\d\\+/,
"etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
@ -113,7 +113,7 @@ Object {
"reason": "Because dogs are cute",
"title": "Dog Pictures",
"updated_at": null,
"url": "https://dogpictures.com",
"url": "https://dogpictures.com/",
},
],
}
@ -123,7 +123,7 @@ exports[`Recommendations Admin API Can browse 2: [headers] 1`] = `
Object {
"access-control-allow-origin": "http://127.0.0.1:2369",
"cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
"content-length": "353",
"content-length": "354",
"content-type": "application/json; charset=utf-8",
"content-version": StringMatching /v\\\\d\\+\\\\\\.\\\\d\\+/,
"etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
@ -159,7 +159,7 @@ Object {
"reason": "Because cats are cute",
"title": "Cat Pictures",
"updated_at": null,
"url": "https://catpictures.com",
"url": "https://catpictures.com/",
},
],
}
@ -169,7 +169,7 @@ exports[`Recommendations Admin API Can edit recommendation 2: [headers] 1`] = `
Object {
"access-control-allow-origin": "http://127.0.0.1:2369",
"cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
"content-length": "354",
"content-length": "355",
"content-type": "application/json; charset=utf-8",
"content-version": StringMatching /v\\\\d\\+\\\\\\.\\\\d\\+/,
"etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,

View File

@ -46,7 +46,7 @@ describe('Recommendations Admin API', function () {
// Check everything is set correctly
assert.equal(body.recommendations[0].title, 'Dog Pictures');
assert.equal(body.recommendations[0].url, 'https://dogpictures.com');
assert.equal(body.recommendations[0].url, 'https://dogpictures.com/');
assert.equal(body.recommendations[0].reason, null);
assert.equal(body.recommendations[0].excerpt, null);
assert.equal(body.recommendations[0].featured_image, null);
@ -84,7 +84,7 @@ describe('Recommendations Admin API', function () {
// Check everything is set correctly
assert.equal(body.recommendations[0].title, 'Dog Pictures');
assert.equal(body.recommendations[0].url, 'https://dogpictures.com');
assert.equal(body.recommendations[0].url, 'https://dogpictures.com/');
assert.equal(body.recommendations[0].reason, 'Because dogs are cute');
assert.equal(body.recommendations[0].excerpt, 'Dogs are cute');
assert.equal(body.recommendations[0].featured_image, 'https://dogpictures.com/dog.jpg');
@ -123,7 +123,7 @@ describe('Recommendations Admin API', function () {
// Check everything is set correctly
assert.equal(body.recommendations[0].id, id);
assert.equal(body.recommendations[0].title, 'Cat Pictures');
assert.equal(body.recommendations[0].url, 'https://catpictures.com');
assert.equal(body.recommendations[0].url, 'https://catpictures.com/');
assert.equal(body.recommendations[0].reason, 'Because cats are cute');
assert.equal(body.recommendations[0].excerpt, 'Cats are cute');
assert.equal(body.recommendations[0].featured_image, 'https://catpictures.com/cat.jpg');

View File

@ -132,7 +132,7 @@ class OEmbedService {
/**
* @param {string} url
*
* @returns {Promise<{url: string, body: string}>}
* @returns {Promise<{url: string, body: string, contentType: string|undefined}>}
*/
async fetchPageHtml(url) {
// Fetch url and get response as binary buffer to
@ -154,7 +154,8 @@ class OEmbedService {
if (encoding === null) {
return {
body: body.toString(),
url: responseUrl
url: responseUrl,
contentType: headers['content-type']
};
}
@ -163,14 +164,16 @@ class OEmbedService {
return {
body: decodedBody,
url: responseUrl
url: responseUrl,
contentType: headers['content-type']
};
} catch (err) {
logging.error(err);
//return non decoded body anyway
return {
body: body.toString(),
url: responseUrl
url: responseUrl,
contentType: headers['content-type']
};
}
}
@ -355,7 +358,7 @@ class OEmbedService {
}
// Not in the list, we need to fetch the content
const {url: pageUrl, body} = await this.fetchPageHtml(url);
const {url: pageUrl, body, contentType} = await this.fetchPageHtml(url);
// fetch only bookmark when explicitly requested
if (type === 'bookmark') {
@ -364,9 +367,27 @@ class OEmbedService {
// mentions need to return bookmark data (metadata) and body (html) for link verification
if (type === 'mention') {
const bookmark = await this.fetchBookmarkData(url, body);
if (contentType.includes('application/json')) {
// No need to fetch metadata: we have none
const bookmark = {
version: '1.0',
type: 'bookmark',
url,
metadata: {
title: null,
description: null,
publisher: null,
author: null,
thumbnail: null,
icon: null
},
contentType
};
return {...bookmark, body};
}
const bookmark = await this.fetchBookmarkData(url, body);
return {...bookmark, body, contentType};
}
// attempt to fetch oembed

View File

@ -22,12 +22,13 @@
"build"
],
"devDependencies": {
"@tryghost/errors": "1.2.24",
"@types/node": "^20.5.7",
"c8": "8.0.1",
"mocha": "10.2.0",
"sinon": "15.2.0",
"ts-node": "10.9.1",
"typescript": "5.2.2",
"@tryghost/errors": "1.2.24"
"typescript": "5.2.2"
},
"dependencies": {}
}

View File

@ -9,7 +9,7 @@ export class InMemoryRecommendationRepository implements RecommendationRepositor
excerpt: "The Pragmatic Programmer is one of those rare tech books youll read, re-read, and read again over the years. Whether youre new to the field or an experienced practitioner, youll come away with fresh insights each and every time.",
featuredImage: "https://www.thepragmaticprogrammer.com/image.png",
favicon: "https://www.shesabeast.co/content/images/size/w256h256/2022/08/transparent-icon-black-copy-gray-bar.png",
url: "https://www.thepragmaticprogrammer.com/",
url: new URL("https://www.thepragmaticprogrammer.com/"),
oneClickSubscribe: false
}),
new Recommendation({
@ -18,7 +18,7 @@ export class InMemoryRecommendationRepository implements RecommendationRepositor
excerpt: "The Pragmatic Programmer is one of those rare tech books youll read, re-read, and read again over the years. Whether youre new to the field or an experienced practitioner, youll come away with fresh insights each and every time.",
featuredImage: "https://www.thepragmaticprogrammer.com/image.png",
favicon: "https://substackcdn.com/image/fetch/f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fc7cde267-8f9e-47fa-9aef-5be03bad95ed%2Fapple-touch-icon-1024x1024.png",
url: "https://www.thepragmaticprogrammer.com/",
url: new URL("https://www.thepragmaticprogrammer.com/"),
oneClickSubscribe: false
}),
new Recommendation({
@ -27,7 +27,7 @@ export class InMemoryRecommendationRepository implements RecommendationRepositor
excerpt: "The Pragmatic Programmer is one of those rare tech books youll read, re-read, and read again over the years. Whether youre new to the field or an experienced practitioner, youll come away with fresh insights each and every time.",
featuredImage: "https://www.thepragmaticprogrammer.com/image.png",
favicon: "https://clickhole.com/wp-content/uploads/2020/05/cropped-clickhole-icon-180x180.png",
url: "https://www.thepragmaticprogrammer.com/",
url: new URL("https://www.thepragmaticprogrammer.com/"),
oneClickSubscribe: false
}),
new Recommendation({
@ -36,7 +36,7 @@ export class InMemoryRecommendationRepository implements RecommendationRepositor
excerpt: "The Pragmatic Programmer is one of those rare tech books youll read, re-read, and read again over the years. Whether youre new to the field or an experienced practitioner, youll come away with fresh insights each and every time.",
featuredImage: "https://www.thepragmaticprogrammer.com/image.png",
favicon: "https://www.theverge.com/icons/apple_touch_icon.png",
url: "https://www.thepragmaticprogrammer.com/",
url: new URL("https://www.thepragmaticprogrammer.com/"),
oneClickSubscribe: false
}),
new Recommendation({
@ -45,7 +45,7 @@ export class InMemoryRecommendationRepository implements RecommendationRepositor
excerpt: "The Pragmatic Programmer is one of those rare tech books youll read, re-read, and read again over the years. Whether youre new to the field or an experienced practitioner, youll come away with fresh insights each and every time.",
featuredImage: "https://www.thepragmaticprogrammer.com/image.png",
favicon: "https://substackcdn.com/image/fetch/w_96,h_96,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff3f2b2ad-681f-45e1-9496-db80f45e853d_403x403.png",
url: "https://www.thepragmaticprogrammer.com/",
url: new URL("https://www.thepragmaticprogrammer.com/"),
oneClickSubscribe: true
})
];

View File

@ -7,12 +7,12 @@ export class Recommendation {
excerpt: string|null // Fetched from the site meta data
featuredImage: string|null // Fetched from the site meta data
favicon: string|null // Fetched from the site meta data
url: string
url: URL
oneClickSubscribe: boolean
createdAt: Date
updatedAt: Date|null
constructor(data: {id?: string, title: string, reason: string|null, excerpt: string|null, featuredImage: string|null, favicon: string|null, url: string, oneClickSubscribe: boolean, createdAt?: Date, updatedAt?: Date|null}) {
constructor(data: {id?: string, title: string, reason: string|null, excerpt: string|null, featuredImage: string|null, favicon: string|null, url: URL, oneClickSubscribe: boolean, createdAt?: Date, updatedAt?: Date|null}) {
this.id = data.id ?? ObjectId().toString();
this.title = data.title;
this.reason = data.reason;

View File

@ -38,6 +38,18 @@ function validateBoolean(object: any, key: string, {required = true} = {}): bool
}
}
function validateURL(object: any, key: string, {required = true} = {}): URL|undefined {
const string = validateString(object, key, {required});
if (string !== undefined) {
try {
return new URL(string);
} catch (e) {
throw new errors.BadRequestError({message: `${key} must be a valid URL`});
}
}
}
export class RecommendationController {
service: RecommendationService;
@ -67,7 +79,7 @@ export class RecommendationController {
const cleanedRecommendation: Omit<Recommendation, 'id'|'createdAt'|'updatedAt'> = {
title: validateString(recommendation, "title") ?? '',
url: validateString(recommendation, "url") ?? '',
url: validateURL(recommendation, "url")!,
// Optional fields
oneClickSubscribe: validateBoolean(recommendation, "one_click_subscribe", {required: false}) ?? false,
@ -89,7 +101,7 @@ export class RecommendationController {
const recommendation = frame.data.recommendations[0];
const cleanedRecommendation: Partial<Recommendation> = {
title: validateString(recommendation, "title", {required: false}),
url: validateString(recommendation, "url", {required: false}),
url: validateURL(recommendation, "url", {required: false}),
oneClickSubscribe: validateBoolean(recommendation, "one_click_subscribe", {required: false}),
reason: validateString(recommendation, "reason", {required: false}),
excerpt: validateString(recommendation, "excerpt", {required: false}),
@ -112,7 +124,7 @@ export class RecommendationController {
excerpt: r.excerpt,
featured_image: r.featuredImage,
favicon: r.favicon,
url: r.url,
url: r.url.toString(),
one_click_subscribe: r.oneClickSubscribe,
created_at: r.createdAt,
updated_at: r.updatedAt

View File

@ -1,26 +1,66 @@
import {Recommendation} from "./Recommendation";
import {RecommendationRepository} from "./RecommendationRepository";
import {WellknownService} from "./WellknownService";
type MentionSendingService = {
sendAll(options: {url: URL, links: URL[]}): Promise<void>
}
export class RecommendationService {
repository: RecommendationRepository;
wellknownService: WellknownService;
mentionSendingService: MentionSendingService;
constructor(deps: {repository: RecommendationRepository}) {
constructor(deps: {repository: RecommendationRepository, wellknownService: WellknownService, mentionSendingService: MentionSendingService}) {
this.repository = deps.repository;
this.wellknownService = deps.wellknownService;
this.mentionSendingService = deps.mentionSendingService;
}
async init() {
await this.updateWellknown();
}
async updateWellknown() {
const recommendations = await this.listRecommendations();
await this.wellknownService.set(recommendations);
}
sendMentionToRecommendation(recommendation: Recommendation) {
this.mentionSendingService.sendAll({
url: this.wellknownService.getURL(),
links: [
recommendation.url
]
}).catch(console.error);
}
async addRecommendation(recommendation: Recommendation) {
return this.repository.add(recommendation);
const r = this.repository.add(recommendation);
await this.updateWellknown();
// Only send an update for the mentioned URL
this.sendMentionToRecommendation(recommendation);
return r;
}
async editRecommendation(id: string, recommendationEdit: Partial<Recommendation>) {
// Check if it exists
const existing = await this.repository.getById(id);
return this.repository.edit(existing.id, recommendationEdit);
const e = await this.repository.edit(existing.id, recommendationEdit);
await this.updateWellknown();
this.sendMentionToRecommendation(e);
return e;
}
async deleteRecommendation(id: string) {
const existing = await this.repository.getById(id);
await this.repository.remove(existing.id);
await this.updateWellknown();
// Send a mention (because it was deleted, according to the webmentions spec)
this.sendMentionToRecommendation(existing);
}
async listRecommendations() {

View File

@ -0,0 +1,48 @@
import {Recommendation} from "./Recommendation"
import fs from "fs/promises";
import _path from 'path';
type UrlUtils = {
relativeToAbsolute(url: string): string
}
type Options = {
/**
* Where to publish the wellknown file
*/
dir: string,
urlUtils: UrlUtils
}
export class WellknownService {
dir: string
urlUtils: UrlUtils
constructor({dir, urlUtils}: Options) {
this.dir = dir;
this.urlUtils = urlUtils;
}
#formatRecommendation(recommendation: Recommendation) {
return {
url: recommendation.url,
reason: recommendation.reason,
updated_at: (recommendation.updatedAt ?? recommendation.createdAt).toISOString(),
created_at: (recommendation.createdAt).toISOString(),
}
}
getPath() {
return _path.join(this.dir, '/.well-known/recommendations.json');
}
getURL(): URL {
return new URL(this.urlUtils.relativeToAbsolute('/.well-known/recommendations.json'));
}
async set(recommendations: Recommendation[]) {
const content = JSON.stringify(recommendations.map(r => this.#formatRecommendation(r)));
const path = this.getPath();
await fs.mkdir(_path.dirname(path), {recursive: true});
await fs.writeFile(path, content);
}
}

View File

@ -3,3 +3,4 @@ export * from './RecommendationService';
export * from './RecommendationRepository';
export * from './InMemoryRecommendationRepository';
export * from './Recommendation';
export * from './WellknownService';

View File

@ -21,11 +21,44 @@ module.exports = class Mention {
/**
* @param {string} html
* @param {string} contentType
*/
verify(html) {
verify(html, contentType) {
const wasVerified = this.#verified;
if (contentType.includes('text/html')) {
try {
const $ = cheerio.load(html);
const hasTargetUrl = $('a[href*="' + this.target.href + '"], img[src*="' + this.target.href + '"], video[src*="' + this.target.href + '"]').length > 0;
this.#verified = hasTargetUrl;
if (wasVerified && !this.#verified) {
// Delete the mention
this.#deleted = true;
this.#verified = true;
}
} catch (e) {
this.#verified = false;
}
}
if (contentType.includes('application/json')) {
try {
// Check valid JSON
JSON.parse(html);
// Check full text string is present in the json
this.#verified = !!html.includes(JSON.stringify(this.target.href));
if (wasVerified && !this.#verified) {
// Delete the mention
this.#deleted = true;
this.#verified = true;
}
} catch (e) {
this.#verified = false;
}
}
}
/** @type {URL} */
@ -274,12 +307,19 @@ module.exports = class Mention {
return mention;
}
/**
* @returns {boolean}
*/
isDeleted() {
return this.#deleted;
}
/**
* @param {Mention} mention
* @returns {boolean}
*/
static isDeleted(mention) {
return mention.#deleted;
return mention.isDeleted();
}
};

View File

@ -67,7 +67,7 @@ module.exports = class MentionSendingService {
let previousHtml = post.previous('status') === 'published' ? post.previous('html') : null;
if (html || previousHtml) {
await this.#jobService.addJob('sendWebmentions', async () => {
await this.sendAll({
await this.sendForHTMLResource({
url: new URL(this.#getPostUrl(post)),
html: html,
previousHtml: previousHtml
@ -80,6 +80,10 @@ module.exports = class MentionSendingService {
}
}
/**
* @param {{source: URL, target: URL, endpoint: URL}} options
* @returns
*/
async send({source, target, endpoint}) {
logging.info('[Webmention] Sending webmention from ' + source.href + ' to ' + target.href + ' via ' + endpoint.href);
@ -113,7 +117,7 @@ module.exports = class MentionSendingService {
* @param {string} resource.html
* @param {string|null} [resource.previousHtml]
*/
async sendAll(resource) {
async sendForHTMLResource(resource) {
const links = resource.html ? this.getLinks(resource.html) : [];
if (resource.previousHtml) {
// We also need to send webmentions for removed links
@ -129,12 +133,25 @@ module.exports = class MentionSendingService {
logging.info('[Webmention] Sending all webmentions for ' + resource.url.href);
}
await this.sendAll({
url: resource.url,
links
});
}
/**
* Send a webmention call for the links in a resource.
* @param {object} resource
* @param {URL} resource.url
* @param {URL[]} resource.links
*/
async sendAll({url, links}) {
for (const target of links) {
const endpoint = await this.#discoveryService.getEndpoint(target);
if (endpoint) {
// Send webmention call
try {
await this.send({source: resource.url, target, endpoint});
await this.send({source: url, target, endpoint});
} catch (e) {
logging.error('[Webmention] Failed sending via ' + endpoint.href + ': ' + e.message);
}

View File

@ -68,13 +68,14 @@ const Mention = require('./Mention');
/**
* @typedef {object} WebmentionMetadata
* @prop {string} siteTitle
* @prop {string} title
* @prop {string} excerpt
* @prop {string} author
* @prop {URL} image
* @prop {URL} favicon
* @prop {string|null} siteTitle
* @prop {string|null} title
* @prop {string|null} excerpt
* @prop {string|null} author
* @prop {URL|null} image
* @prop {URL|null} favicon
* @prop {string} body
* @prop {string|undefined} contentType
*/
/**
@ -228,7 +229,7 @@ module.exports = class MentionsAPI {
if (metadata?.body) {
try {
mention.verify(metadata.body);
mention.verify(metadata.body, metadata.contentType);
} catch (e) {
logging.error(e);
}

View File

@ -1,6 +1,8 @@
const assert = require('assert/strict');
const ObjectID = require('bson-objectid');
const Mention = require('../lib/Mention');
const cheerio = require('cheerio');
const sinon = require('sinon');
const validInput = {
source: 'https://source.com',
@ -35,15 +37,40 @@ describe('Mention', function () {
});
describe('verify', function () {
afterEach(function () {
sinon.restore();
});
it('can handle invalid HTML', async function () {
const mention = await Mention.create(validInput);
assert(!mention.verified);
sinon.stub(cheerio, 'load').throws(new Error('Invalid HTML'));
mention.verify('irrelevant', 'text/html');
assert(!mention.verified);
assert(!mention.isDeleted());
});
it('Does basic check for the target URL and updates verified property', async function () {
const mention = await Mention.create(validInput);
assert(!mention.verified);
mention.verify('<a href="https://target.com/">');
mention.verify('<a href="https://target.com/">', 'text/html');
assert(mention.verified);
assert(!mention.isDeleted());
mention.verify('<a href="https://not-da-target.com">');
mention.verify('something else', 'text/html');
assert(mention.verified);
assert(mention.isDeleted());
});
it('detects differences', async function () {
const mention = await Mention.create(validInput);
assert(!mention.verified);
mention.verify('<a href="https://not-target.com/">', 'text/html');
assert(!mention.verified);
assert(!mention.isDeleted());
});
it('Does check for Image targets', async function () {
const mention = await Mention.create({
@ -52,11 +79,13 @@ describe('Mention', function () {
});
assert(!mention.verified);
mention.verify('<img src="https://target.com/image.jpg">');
mention.verify('<img src="https://target.com/image.jpg">', 'text/html');
assert(mention.verified);
assert(!mention.isDeleted());
mention.verify('<img src="https://not-da-target.com/image.jpg">');
assert(!mention.verified);
mention.verify('something else', 'text/html');
assert(mention.verified);
assert(mention.isDeleted());
});
it('Does check for Video targets', async function () {
const mention = await Mention.create({
@ -65,11 +94,35 @@ describe('Mention', function () {
});
assert(!mention.verified);
mention.verify('<video src="https://target.com/video.mp4">');
mention.verify('<video src="https://target.com/video.mp4">', 'text/html');
assert(mention.verified);
assert(!mention.isDeleted());
mention.verify('<video src="https://not-da-target.com/video.mp4">');
mention.verify('something else', 'text/html');
assert(mention.verified);
assert(mention.isDeleted());
});
it('can verify links in JSON', async function () {
const mention = await Mention.create(validInput);
assert(!mention.verified);
mention.verify('{"url": "https://target.com/"}', 'application/json');
assert(mention.verified);
assert(!mention.isDeleted());
mention.verify('{}', 'application/json');
assert(mention.verified);
assert(mention.isDeleted());
});
it('can handle invalid JSON', async function () {
const mention = await Mention.create(validInput);
assert(!mention.verified);
mention.verify('{"url": "ht', 'application/json');
assert(!mention.verified);
assert(!mention.isDeleted());
});
});

View File

@ -55,7 +55,7 @@ describe('MentionSendingService', function () {
const service = new MentionSendingService({
isEnabled: () => false
});
const stub = sinon.stub(service, 'sendAll');
const stub = sinon.stub(service, 'sendForHTMLResource');
await service.sendForPost({});
sinon.assert.notCalled(stub);
});
@ -64,7 +64,7 @@ describe('MentionSendingService', function () {
const service = new MentionSendingService({
isEnabled: () => true
});
const stub = sinon.stub(service, 'sendAll');
const stub = sinon.stub(service, 'sendForHTMLResource');
let options = {importing: true};
await service.sendForPost({}, options);
sinon.assert.notCalled(stub);
@ -74,7 +74,7 @@ describe('MentionSendingService', function () {
const service = new MentionSendingService({
isEnabled: () => true
});
const stub = sinon.stub(service, 'sendAll');
const stub = sinon.stub(service, 'sendForHTMLResource');
let options = {context: {internal: true}};
await service.sendForPost({}, options);
sinon.assert.notCalled(stub);
@ -84,7 +84,7 @@ describe('MentionSendingService', function () {
const service = new MentionSendingService({
isEnabled: () => true
});
const stub = sinon.stub(service, 'sendAll');
const stub = sinon.stub(service, 'sendForHTMLResource');
await service.sendForPost(createModel({
status: 'draft',
html: 'changed',
@ -100,7 +100,7 @@ describe('MentionSendingService', function () {
const service = new MentionSendingService({
isEnabled: () => true
});
const stub = sinon.stub(service, 'sendAll');
const stub = sinon.stub(service, 'sendForHTMLResource');
await service.sendForPost(createModel({
status: 'published',
html: 'same',
@ -116,7 +116,7 @@ describe('MentionSendingService', function () {
const service = new MentionSendingService({
isEnabled: () => true
});
const stub = sinon.stub(service, 'sendAll');
const stub = sinon.stub(service, 'sendForHTMLResource');
await service.sendForPost(createModel({
status: 'send',
html: 'changed',
@ -134,7 +134,7 @@ describe('MentionSendingService', function () {
getPostUrl: () => 'https://site.com/post/',
jobService: jobService
});
const stub = sinon.stub(service, 'sendAll');
const stub = sinon.stub(service, 'sendForHTMLResource');
await service.sendForPost(createModel({
status: 'published',
html: 'same',
@ -156,7 +156,7 @@ describe('MentionSendingService', function () {
getPostUrl: () => 'https://site.com/post/',
jobService: jobService
});
const stub = sinon.stub(service, 'sendAll');
const stub = sinon.stub(service, 'sendForHTMLResource');
await service.sendForPost(createModel({
status: 'published',
html: 'updated',
@ -177,7 +177,7 @@ describe('MentionSendingService', function () {
isEnabled: () => true,
getPostUrl: () => 'https://site.com/post/'
});
sinon.stub(service, 'sendAll').rejects(new Error('Internal error test'));
sinon.stub(service, 'sendForHTMLResource').rejects(new Error('Internal error test'));
await service.sendForPost(createModel({
status: 'published',
html: 'same',
@ -195,7 +195,7 @@ describe('MentionSendingService', function () {
getPostUrl: () => 'https://site.com/post/',
jobService: jobService
});
const stub = sinon.stub(service, 'sendAll');
const stub = sinon.stub(service, 'sendForHTMLResource');
await service.sendForPost(createModel({
status: 'published',
html: '',
@ -208,7 +208,7 @@ describe('MentionSendingService', function () {
});
});
describe('sendAll', function () {
describe('sendForHTMLResource', function () {
it('Sends to all links', async function () {
this.retries(1);
let counter = 0;
@ -227,7 +227,7 @@ describe('MentionSendingService', function () {
getEndpoint: async () => new URL('https://example.org/webmentions-test')
}
});
await service.sendAll({url: new URL('https://site.com'),
await service.sendForHTMLResource({url: new URL('https://site.com'),
html: `
<html>
<body>
@ -263,7 +263,7 @@ describe('MentionSendingService', function () {
getEndpoint: async () => new URL('https://example.org/webmentions-test')
}
});
await service.sendAll({url: new URL('https://site.com'),
await service.sendForHTMLResource({url: new URL('https://site.com'),
html: `
<html>
<body>
@ -298,7 +298,7 @@ describe('MentionSendingService', function () {
},
jobService: jobService
});
await service.sendAll({url: new URL('https://site.com'),
await service.sendForHTMLResource({url: new URL('https://site.com'),
html: `<a href="https://example.com">Example</a>`,
previousHtml: `<a href="https://typo.com">Example</a>`});
assert.equal(scope.isDone(), true);
@ -311,7 +311,7 @@ describe('MentionSendingService', function () {
isEnabled: () => true
});
const linksStub = sinon.stub(service, 'getLinks');
await service.sendAll({html: ``,previousHtml: ``});
await service.sendForHTMLResource({html: ``,previousHtml: ``});
sinon.assert.notCalled(linksStub);
});
});

View File

@ -8588,7 +8588,7 @@
dependencies:
"@types/node" "*"
"@types/node@*", "@types/node@>=10.0.0", "@types/node@>=12.12.47", "@types/node@>=13.7.0", "@types/node@>=8.1.0":
"@types/node@*", "@types/node@>=10.0.0", "@types/node@>=12.12.47", "@types/node@>=13.7.0", "@types/node@>=8.1.0", "@types/node@^20.5.7":
version "20.5.7"
resolved "https://registry.yarnpkg.com/@types/node/-/node-20.5.7.tgz#4b8ecac87fbefbc92f431d09c30e176fc0a7c377"
integrity sha512-dP7f3LdZIysZnmvP3ANJYTSwg+wLLl8p7RqniVlV7j+oXSXAbt9h0WIBFmJy5inWZoX9wZN6eXx+YXd9Rh3RBA==