TryGhost/Ghost
1
0
Fork 0
mirror of https://github.com/TryGhost/Ghost.git synced 2024-10-05 09:18:03 +03:00
Code Issues Projects Releases Wiki Activity

🏗 Changed internal URL storage format to use __GHOST_URL__ (#12731)

Browse Source 
closes https://github.com/TryGhost/Team/issues/467

- switches to storing "transform-ready" URLs in the database
- transform-ready URLs contain a `__GHOST_URL__` placeholder that corresponds to the configured url that gives a few benefits
  - much faster and less memory intensive output transformations through not needing to parse html or markdown - the transform can be achieved using a straightforward regex find+replace
  - ability to change to/from or rename subdirectory without any manual updates to the database
- modified existing 4.0 url-transformation migration rather than adding another one and repeating the transformation on posts rows
This commit is contained in:
Kevin Ansfield 2021-03-05 13:54:01 +00:00 committed by GitHub
parent b4140d4310
commit a6f5eb71be
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 439 additions and 338 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/frontend/services/rss/generate-feed.js
View File

@ -20,7 +20,7 @@ const generateTags = function generateTags(data) {
const generateItem = function generateItem(post, secure) {
const itemUrl = urlService.getUrlByResourceId(post.id, {secure, absolute: true});
const htmlContent = cheerio.load(urlUtils.htmlRelativeToAbsolute(post.html, itemUrl, {secure}) || '', {decodeEntities: false});
const htmlContent = cheerio.load(post.html || '');
const item = {
title: post.title,
// @TODO: DRY this up with data/meta/index & other excerpt code

19
core/server/api/canary/utils/serializers/input/utils/url.js
View File

@ -1,15 +1,20 @@
const urlUtils = require('../../../../../../../shared/url-utils');
const handleImageUrl = (imageUrl) => {
const blogDomain = urlUtils.getSiteUrl().replace(/^http(s?):\/\//, '').replace(/\/$/, '');
const imageUrlAbsolute = imageUrl.replace(/^http(s?):\/\//, '');
const imagePathRe = new RegExp(`^${blogDomain}/${urlUtils.STATIC_IMAGE_URL_PREFIX}`);
try {
const imageURL = new URL(imageUrl, urlUtils.getSiteUrl());
const siteURL = new URL(urlUtils.getSiteUrl());
const subdir = siteURL.pathname.replace(/\/$/, '');
const imagePathRe = new RegExp(`${subdir}/${urlUtils.STATIC_IMAGE_URL_PREFIX}`);
if (imagePathRe.test(imageUrlAbsolute)) {
return urlUtils.absoluteToRelative(imageUrl);
if (imagePathRe.test(imageURL.pathname)) {
return urlUtils.toTransformReady(imageUrl);
}
return imageUrl;
} catch (e) {
return imageUrl;
}
return imageUrl;
};
const forPost = (attrs, options) => {

4
core/server/api/canary/utils/serializers/output/snippets.js
View File

@ -1,5 +1,6 @@
//@ts-check
const debug = require('ghost-ignition').debug('api:canary:utils:serializers:output:snippets');
const urlUtils = require('../../../../../../shared/url-utils');
module.exports = {
browse: createSerializer('browse', paginatedSnippets),
@ -65,7 +66,8 @@ function serializeSnippet(snippet, options) {
return {
id: json.id,
name: json.name,
mobiledoc: json.mobiledoc,
// @ts-ignore
mobiledoc: urlUtils.transformReadyToAbsolute(json.mobiledoc),
created_at: json.created_at,
updated_at: json.updated_at,
created_by: json.created_by,

58
core/server/api/canary/utils/serializers/output/utils/url.js
View File

@ -30,26 +30,10 @@ const forPost = (id, attrs, frame) => {
}
}
if (attrs.mobiledoc) {
attrs.mobiledoc = urlUtils.mobiledocRelativeToAbsolute(
attrs.mobiledoc,
attrs.url
);
}
['html', 'codeinjection_head', 'codeinjection_foot'].forEach((attr) => {
if (attrs[attr]) {
attrs[attr] = urlUtils.htmlRelativeToAbsolute(
attrs[attr],
attrs.url
);
}
});
['feature_image', 'canonical_url', 'posts_meta.og_image', 'posts_meta.twitter_image'].forEach((path) => {
['mobiledoc', 'html', 'plaintext', 'codeinjection_head', 'codeinjection_foot', 'feature_image', 'canonical_url', 'posts_meta.og_image', 'posts_meta.twitter_image'].forEach((path) => {
const value = _.get(attrs, path);
if (value) {
_.set(attrs, path, urlUtils.relativeToAbsolute(value));
_.set(attrs, path, urlUtils.transformReadyToAbsolute(value));
}
});
@ -65,13 +49,11 @@ const forUser = (id, attrs, options) => {
attrs.url = urlService.getUrlByResourceId(id, {absolute: true});
}
if (attrs.profile_image) {
attrs.profile_image = urlUtils.urlFor('image', {image: attrs.profile_image}, true);
}
if (attrs.cover_image) {
attrs.cover_image = urlUtils.urlFor('image', {image: attrs.cover_image}, true);
}
['profile_image', 'cover_image'].forEach((attr) => {
if (attrs[attr]) {
attrs[attr] = urlUtils.transformReadyToAbsolute(attrs[attr]);
}
});
return attrs;
};
@ -81,9 +63,11 @@ const forTag = (id, attrs, options) => {
attrs.url = urlService.getUrlByResourceId(id, {absolute: true});
}
if (attrs.feature_image) {
attrs.feature_image = urlUtils.urlFor('image', {image: attrs.feature_image}, true);
}
['feature_image', 'og_image', 'twitter_image', 'codeinjection_head', 'codeinjection_foot'].forEach((attr) => {
if (attrs[attr]) {
attrs[attr] = urlUtils.transformReadyToAbsolute(attrs[attr]);
}
});
return attrs;
};
@ -94,21 +78,15 @@ const forSettings = (attrs) => {
if (_.isArray(attrs)) {
attrs.forEach((obj) => {
if (['cover_image', 'logo', 'icon', 'portal_button_icon'].includes(obj.key) && obj.value) {
obj.value = urlUtils.urlFor('image', {image: obj.value}, true);
obj.value = urlUtils.transformReadyToAbsolute(obj.value);
}
});
} else {
if (attrs.cover_image) {
attrs.cover_image = urlUtils.urlFor('image', {image: attrs.cover_image}, true);
}
if (attrs.logo) {
attrs.logo = urlUtils.urlFor('image', {image: attrs.logo}, true);
}
if (attrs.icon) {
attrs.icon = urlUtils.urlFor('image', {image: attrs.icon}, true);
}
['cover_image', 'logo', 'icon', 'portal_button_icon'].forEach((attr) => {
if (attrs[attr]) {
attrs[attr] = urlUtils.transformReadyToAbsolute(attrs[attr]);
}
});
}
return attrs;

19
core/server/api/v2/utils/serializers/input/utils/url.js
View File

@ -1,15 +1,20 @@
const urlUtils = require('../../../../../../../shared/url-utils');
const handleImageUrl = (imageUrl) => {
const siteDomain = urlUtils.getSiteUrl().replace(/^http(s?):\/\//, '').replace(/\/$/, '');
const imageUrlAbsolute = imageUrl.replace(/^http(s?):\/\//, '');
const imagePathRe = new RegExp(`^${siteDomain}/${urlUtils.STATIC_IMAGE_URL_PREFIX}`);
try {
const imageURL = new URL(imageUrl, urlUtils.getSiteUrl());
const siteURL = new URL(urlUtils.getSiteUrl());
const subdir = siteURL.pathname.replace(/\/$/, '');
const imagePathRe = new RegExp(`${subdir}/${urlUtils.STATIC_IMAGE_URL_PREFIX}`);
if (imagePathRe.test(imageUrlAbsolute)) {
return urlUtils.absoluteToRelative(imageUrl);
if (imagePathRe.test(imageURL.pathname)) {
return urlUtils.toTransformReady(imageUrl);
}
return imageUrl;
} catch (e) {
return imageUrl;
}
return imageUrl;
};
const forPost = (attrs, options) => {

60
core/server/api/v2/utils/serializers/output/utils/url.js
View File

@ -38,28 +38,10 @@ const forPost = (id, attrs, frame) => {
urlOptions.assetsOnly = true;
}
if (attrs.mobiledoc) {
attrs.mobiledoc = urlUtils.mobiledocRelativeToAbsolute(
attrs.mobiledoc,
attrs.url,
urlOptions
);
}
['html', 'codeinjection_head', 'codeinjection_foot'].forEach((attr) => {
if (attrs[attr]) {
attrs[attr] = urlUtils.htmlRelativeToAbsolute(
attrs[attr],
attrs.url,
urlOptions
);
}
});
['feature_image', 'canonical_url', 'posts_meta.og_image', 'posts_meta.twitter_image'].forEach((path) => {
['mobiledoc', 'html', 'plaintext', 'codeinjection_head', 'codeinjection_foot', 'feature_image', 'canonical_url', 'posts_meta.og_image', 'posts_meta.twitter_image'].forEach((path) => {
const value = _.get(attrs, path);
if (value) {
_.set(attrs, path, urlUtils.relativeToAbsolute(value));
_.set(attrs, path, urlUtils.transformReadyToAbsolute(value, urlOptions));
}
});
@ -75,13 +57,11 @@ const forUser = (id, attrs, options) => {
attrs.url = urlService.getUrlByResourceId(id, {absolute: true});
}
if (attrs.profile_image) {
attrs.profile_image = urlUtils.urlFor('image', {image: attrs.profile_image}, true);
}
if (attrs.cover_image) {
attrs.cover_image = urlUtils.urlFor('image', {image: attrs.cover_image}, true);
}
['profile_image', 'cover_image'].forEach((attr) => {
if (attrs[attr]) {
attrs[attr] = urlUtils.transformReadyToAbsolute(attrs[attr]);
}
});
return attrs;
};
@ -91,9 +71,11 @@ const forTag = (id, attrs, options) => {
attrs.url = urlService.getUrlByResourceId(id, {absolute: true});
}
if (attrs.feature_image) {
attrs.feature_image = urlUtils.urlFor('image', {image: attrs.feature_image}, true);
}
['feature_image', 'og_image', 'twitter_image', 'codeinjection_head', 'codeinjection_foot'].forEach((attr) => {
if (attrs[attr]) {
attrs[attr] = urlUtils.transformReadyToAbsolute(attrs[attr]);
}
});
return attrs;
};
@ -104,21 +86,15 @@ const forSettings = (attrs) => {
if (_.isArray(attrs)) {
attrs.forEach((obj) => {
if (['cover_image', 'logo', 'icon'].includes(obj.key) && obj.value) {
obj.value = urlUtils.urlFor('image', {image: obj.value}, true);
obj.value = urlUtils.transformReadyToAbsolute(obj.value);
}
});
} else {
if (attrs.cover_image) {
attrs.cover_image = urlUtils.urlFor('image', {image: attrs.cover_image}, true);
}
if (attrs.logo) {
attrs.logo = urlUtils.urlFor('image', {image: attrs.logo}, true);
}
if (attrs.icon) {
attrs.icon = urlUtils.urlFor('image', {image: attrs.icon}, true);
}
['cover_image', 'logo', 'icon'].forEach((attr) => {
if (attrs[attr]) {
attrs[attr] = urlUtils.transformReadyToAbsolute(attrs[attr]);
}
});
}
return attrs;

19
core/server/api/v3/utils/serializers/input/utils/url.js
View File

@ -1,15 +1,20 @@
const urlUtils = require('../../../../../../../shared/url-utils');
const handleImageUrl = (imageUrl) => {
const blogDomain = urlUtils.getSiteUrl().replace(/^http(s?):\/\//, '').replace(/\/$/, '');
const imageUrlAbsolute = imageUrl.replace(/^http(s?):\/\//, '');
const imagePathRe = new RegExp(`^${blogDomain}/${urlUtils.STATIC_IMAGE_URL_PREFIX}`);
try {
const imageURL = new URL(imageUrl, urlUtils.getSiteUrl());
const siteURL = new URL(urlUtils.getSiteUrl());
const subdir = siteURL.pathname.replace(/\/$/, '');
const imagePathRe = new RegExp(`${subdir}/${urlUtils.STATIC_IMAGE_URL_PREFIX}`);
if (imagePathRe.test(imageUrlAbsolute)) {
return urlUtils.absoluteToRelative(imageUrl);
if (imagePathRe.test(imageURL.pathname)) {
return urlUtils.toTransformReady(imageUrl);
}
return imageUrl;
} catch (e) {
return imageUrl;
}
return imageUrl;
};
const forPost = (attrs, options) => {

4
core/server/api/v3/utils/serializers/output/snippets.js
View File

@ -1,5 +1,6 @@
//@ts-check
const debug = require('ghost-ignition').debug('api:v3:utils:serializers:output:snippets');
const urlUtils = require('../../../../../../shared/url-utils');
module.exports = {
browse: createSerializer('browse', paginatedSnippets),
@ -65,7 +66,8 @@ function serializeSnippet(snippet, options) {
return {
id: json.id,
name: json.name,
mobiledoc: json.mobiledoc,
// @ts-ignore
mobiledoc: urlUtils.transformReadyToAbsolute(json.mobiledoc),
created_at: json.created_at,
updated_at: json.updated_at,
created_by: json.created_by,

58
core/server/api/v3/utils/serializers/output/utils/url.js
View File

@ -30,26 +30,10 @@ const forPost = (id, attrs, frame) => {
}
}
if (attrs.mobiledoc) {
attrs.mobiledoc = urlUtils.mobiledocRelativeToAbsolute(
attrs.mobiledoc,
attrs.url
);
}
['html', 'codeinjection_head', 'codeinjection_foot'].forEach((attr) => {
if (attrs[attr]) {
attrs[attr] = urlUtils.htmlRelativeToAbsolute(
attrs[attr],
attrs.url
);
}
});
['feature_image', 'canonical_url', 'posts_meta.og_image', 'posts_meta.twitter_image'].forEach((path) => {
['mobiledoc', 'html', 'plaintext', 'codeinjection_head', 'codeinjection_foot', 'feature_image', 'canonical_url', 'posts_meta.og_image', 'posts_meta.twitter_image'].forEach((path) => {
const value = _.get(attrs, path);
if (value) {
_.set(attrs, path, urlUtils.relativeToAbsolute(value));
_.set(attrs, path, urlUtils.transformReadyToAbsolute(value));
}
});
@ -65,13 +49,11 @@ const forUser = (id, attrs, options) => {
attrs.url = urlService.getUrlByResourceId(id, {absolute: true});
}
if (attrs.profile_image) {
attrs.profile_image = urlUtils.urlFor('image', {image: attrs.profile_image}, true);
}
if (attrs.cover_image) {
attrs.cover_image = urlUtils.urlFor('image', {image: attrs.cover_image}, true);
}
['profile_image', 'cover_image'].forEach((attr) => {
if (attrs[attr]) {
attrs[attr] = urlUtils.transformReadyToAbsolute(attrs[attr]);
}
});
return attrs;
};
@ -81,9 +63,11 @@ const forTag = (id, attrs, options) => {
attrs.url = urlService.getUrlByResourceId(id, {absolute: true});
}
if (attrs.feature_image) {
attrs.feature_image = urlUtils.urlFor('image', {image: attrs.feature_image}, true);
}
['feature_image', 'og_image', 'twitter_image', 'codeinjection_head', 'codeinjection_foot'].forEach((attr) => {
if (attrs[attr]) {
attrs[attr] = urlUtils.transformReadyToAbsolute(attrs[attr]);
}
});
return attrs;
};
@ -94,21 +78,15 @@ const forSettings = (attrs) => {
if (_.isArray(attrs)) {
attrs.forEach((obj) => {
if (['cover_image', 'logo', 'icon', 'portal_button_icon'].includes(obj.key) && obj.value) {
obj.value = urlUtils.urlFor('image', {image: obj.value}, true);
obj.value = urlUtils.transformReadyToAbsolute(obj.value);
}
});
} else {
if (attrs.cover_image) {
attrs.cover_image = urlUtils.urlFor('image', {image: attrs.cover_image}, true);
}
if (attrs.logo) {
attrs.logo = urlUtils.urlFor('image', {image: attrs.logo}, true);
}
if (attrs.icon) {
attrs.icon = urlUtils.urlFor('image', {image: attrs.icon}, true);
}
['cover_image', 'logo', 'icon', 'portal_button_icon'].forEach((attr) => {
if (attrs[attr]) {
attrs[attr] = urlUtils.transformReadyToAbsolute(attrs[attr]);
}
});
}
return attrs;

77
core/server/data/migrations/versions/4.0/18-transform-posts-absolute-to-relative.js
View File

@ -1,77 +0,0 @@
const logging = require('../../../../../shared/logging');
const urlUtils = require('../../../../../shared/url-utils');
const {createIrreversibleMigration} = require('../../utils');
module.exports = createIrreversibleMigration(async (knex) => {
logging.info('Transforming all internal urls in posts from absolute to relative');
await knex.transaction(async (trx) => {
// get list of posts ids, use .forUpdate to lock rows until the transaction is finished
const postIdRows = await knex('posts')
.transacting(trx)
.forUpdate()
.select('id');
// transform each post individually to avoid dumping all posts into memory and
// pushing all queries into the query builder buffer in parallel
// https://stackoverflow.com/questions/54105280/how-to-loop-through-multi-line-sql-query-and-use-them-in-knex-transactions
for (const postIdRow of postIdRows) {
const {id} = postIdRow;
const [post] = await knex('posts')
.transacting(trx)
.where({id})
.select([
'mobiledoc',
'custom_excerpt',
'codeinjection_head',
'codeinjection_foot',
'feature_image',
'canonical_url'
]);
/* eslint-disable camelcase */
const mobiledoc = urlUtils.mobiledocAbsoluteToRelative(post.mobiledoc);
const custom_excerpt = urlUtils.htmlAbsoluteToRelative(post.custom_excerpt);
const codeinjection_head = urlUtils.htmlAbsoluteToRelative(post.codeinjection_head);
const codeinjection_foot = urlUtils.htmlAbsoluteToRelative(post.codeinjection_foot);
const feature_image = urlUtils.absoluteToRelative(post.feature_image);
const canonical_url = urlUtils.absoluteToRelative(post.canonical_url, {ignoreProtocol: false});
await knex('posts')
.transacting(trx)
.where({id})
.update({
mobiledoc,
custom_excerpt,
codeinjection_head,
codeinjection_foot,
feature_image,
canonical_url
});
const [postMeta] = await knex('posts_meta')
.transacting(trx)
.where({post_id: id})
.select([
'og_image',
'twitter_image'
]);
if (postMeta) {
const og_image = urlUtils.absoluteToRelative(postMeta.og_image);
const twitter_image = urlUtils.absoluteToRelative(postMeta.twitter_image);
await knex('posts_meta')
.transacting(trx)
.where({post_id: id})
.update({
og_image,
twitter_image
});
}
}
return 'transaction complete';
});
});

195
core/server/data/migrations/versions/4.0/18-transform-urls-absolute-to-transform-ready.js Normal file
View File

@ -0,0 +1,195 @@
const logging = require('../../../../../shared/logging');
const urlUtils = require('../../../../../shared/url-utils');
const {createIrreversibleMigration} = require('../../utils');
module.exports = createIrreversibleMigration(async (knex) => {
logging.info('Transforming all internal urls to transform-ready');
await knex.transaction(async (trx) => {
// posts and posts_meta
// get list of posts ids, use .forUpdate to lock rows until the transaction is finished
const postIdRows = await knex('posts')
.transacting(trx)
.forUpdate()
.select('id');
// transform each post individually to avoid dumping all posts into memory and
// pushing all queries into the query builder buffer in parallel
// https://stackoverflow.com/questions/54105280/how-to-loop-through-multi-line-sql-query-and-use-them-in-knex-transactions
for (const postIdRow of postIdRows) {
const {id} = postIdRow;
const [post] = await knex('posts')
.transacting(trx)
.where({id})
.select([
'mobiledoc',
'custom_excerpt',
'codeinjection_head',
'codeinjection_foot',
'feature_image',
'canonical_url'
]);
/* eslint-disable camelcase */
const mobiledoc = urlUtils.mobiledocToTransformReady(post.mobiledoc);
const custom_excerpt = urlUtils.htmlToTransformReady(post.custom_excerpt);
const codeinjection_head = urlUtils.htmlToTransformReady(post.codeinjection_head);
const codeinjection_foot = urlUtils.htmlToTransformReady(post.codeinjection_foot);
const feature_image = urlUtils.toTransformReady(post.feature_image);
const canonical_url = urlUtils.toTransformReady(post.canonical_url, {ignoreProtocol: false});
await knex('posts')
.transacting(trx)
.where({id})
.update({
mobiledoc,
custom_excerpt,
codeinjection_head,
codeinjection_foot,
feature_image,
canonical_url
});
const [postMeta] = await knex('posts_meta')
.transacting(trx)
.where({post_id: id})
.select([
'og_image',
'twitter_image'
]);
if (postMeta) {
const og_image = urlUtils.toTransformReady(postMeta.og_image);
const twitter_image = urlUtils.toTransformReady(postMeta.twitter_image);
await knex('posts_meta')
.transacting(trx)
.where({post_id: id})
.update({
og_image,
twitter_image
});
}
}
// users
const userIdRows = await knex('users')
.transacting(trx)
.forUpdate()
.select('id');
for (const userIdRow of userIdRows) {
const {id} = userIdRow;
const [user] = await knex('users')
.transacting(trx)
.where({id})
.select([
'profile_image',
'cover_image'
]);
const profile_image = urlUtils.toTransformReady(user.profile_image);
const cover_image = urlUtils.toTransformReady(user.cover_image);
await knex('users')
.transacting(trx)
.where({id})
.update({
profile_image,
cover_image
});
}
// tags
const tagIdRows = await knex('tags')
.transacting(trx)
.forUpdate()
.select('id');
for (const tagIdRow of tagIdRows) {
const {id} = tagIdRow;
const [tag] = await knex('tags')
.transacting(trx)
.where({id})
.select([
'feature_image',
'og_image',
'twitter_image',
'codeinjection_head',
'codeinjection_foot',
'canonical_url'
]);
const feature_image = urlUtils.toTransformReady(tag.feature_image);
const og_image = urlUtils.toTransformReady(tag.og_image);
const twitter_image = urlUtils.toTransformReady(tag.twitter_image);
const codeinjection_head = urlUtils.htmlToTransformReady(tag.codeinjection_head);
const codeinjection_foot = urlUtils.htmlToTransformReady(tag.codeinjection_foot);
const canonical_url = urlUtils.toTransformReady(tag.canonical_url, {ignoreProtocol: false});
await knex('tags')
.transacting(trx)
.where({id})
.update({
feature_image,
og_image,
twitter_image,
codeinjection_head,
codeinjection_foot,
canonical_url
});
}
// snippets
const snippetIdRows = await knex('snippets')
.transacting(trx)
.forUpdate()
.select('id');
for (const snippetIdRow of snippetIdRows) {
const {id} = snippetIdRow;
const [snippet] = await knex('snippets')
.transacting(trx)
.where({id})
.select([
'mobiledoc'
]);
const mobiledoc = urlUtils.mobiledocToTransformReady(snippet.mobiledoc);
await knex('snippets')
.transacting(trx)
.where({id})
.update({
mobiledoc
});
}
// settings
const settingsRows = await knex('settings')
.transacting(trx)
.forUpdate()
.whereIn('key', [
'cover_image',
'logo',
'icon',
'portal_button_icon',
'og_image',
'twitter_image'
]);
for (const settingRow of settingsRows) {
let {key, value} = settingRow;
value = urlUtils.toTransformReady(value);
await knex('settings')
.transacting(trx)
.where({key})
.update({value});
}
return 'transaction complete';
});
});

18
core/server/models/post.js
View File

@ -419,18 +419,18 @@ Post = ghostBookshelf.Model.extend({
this.set('mobiledoc', JSON.stringify(mobiledocLib.blankDocument));
}
// ensure all URLs are stored as relative
// ensure all URLs are stored as transform-ready with __GHOST_URL__ representing config.url
// note: html is not necessary to change because it's a generated later from mobiledoc
const urlTransformMap = {
mobiledoc: 'mobiledocAbsoluteToRelative',
custom_excerpt: 'htmlAbsoluteToRelative',
codeinjection_head: 'htmlAbsoluteToRelative',
codeinjection_foot: 'htmlAbsoluteToRelative',
feature_image: 'absoluteToRelative',
og_image: 'absoluteToRelative',
twitter_image: 'absoluteToRelative',
mobiledoc: 'mobiledocToTransformReady',
custom_excerpt: 'htmlToTransformReady',
codeinjection_head: 'htmlToTransformReady',
codeinjection_foot: 'htmlToTransformReady',
feature_image: 'toTransformReady',
og_image: 'toTransformReady',
twitter_image: 'toTransformReady',
canonical_url: {
method: 'absoluteToRelative',
method: 'toTransformReady',
options: {
ignoreProtocol: false
}

4
core/server/models/posts-meta.js
View File

@ -6,8 +6,8 @@ const PostsMeta = ghostBookshelf.Model.extend({
onSaving: function onSaving() {
const urlTransformMap = {
og_image: 'absoluteToRelative',
twitter_image: 'absoluteToRelative'
og_image: 'toTransformReady',
twitter_image: 'toTransformReady'
};
Object.entries(urlTransformMap).forEach(([attr, transform]) => {

26
core/server/models/snippet.js
View File

@ -1,7 +1,31 @@
const ghostBookshelf = require('./base');
const urlUtils = require('../../shared/url-utils');
const Snippet = ghostBookshelf.Model.extend({
tableName: 'snippets'
tableName: 'snippets',
onSaving: function onSaving() {
const urlTransformMap = {
mobiledoc: 'mobiledocToTransformReady'
};
Object.entries(urlTransformMap).forEach(([attr, transform]) => {
let method = transform;
let methodOptions = {};
if (typeof transform === 'object') {
method = transform.method;
methodOptions = transform.options || {};
}
if (this.hasChanged(attr) && this.get(attr)) {
const transformedValue = urlUtils[method](this.get(attr), methodOptions);
this.set(attr, transformedValue);
}
});
ghostBookshelf.Model.prototype.onSaving.apply(this, arguments);
}
});
const Snippets = ghostBookshelf.Collection.extend({

30
core/server/models/tag.js
View File

@ -1,6 +1,7 @@
const ghostBookshelf = require('./base');
const {i18n} = require('../lib/common');
const errors = require('@tryghost/errors');
const urlUtils = require('../../shared/url-utils');
let Tag;
let Tags;
@ -41,6 +42,35 @@ Tag = ghostBookshelf.Model.extend({
onSaving: function onSaving(newTag, attr, options) {
const self = this;
const urlTransformMap = {
feature_image: 'toTransformReady',
og_image: 'toTransformReady',
twitter_image: 'toTransformReady',
codeinjection_head: 'htmlToTransformReady',
codeinjection_foot: 'htmlToTransformReady',
canonical_url: {
method: 'toTransformReady',
options: {
ignoreProtocol: false
}
}
};
Object.entries(urlTransformMap).forEach(([urlAttr, transform]) => {
let method = transform;
let methodOptions = {};
if (typeof transform === 'object') {
method = transform.method;
methodOptions = transform.options || {};
}
if (this.hasChanged(urlAttr) && this.get(urlAttr)) {
const transformedValue = urlUtils[method](this.get(urlAttr), methodOptions);
this.set(urlAttr, transformedValue);
}
});
ghostBookshelf.Model.prototype.onSaving.apply(this, arguments);
// name: #later slug: hash-later

21
core/server/models/user.js
View File

@ -12,6 +12,7 @@ const {gravatar} = require('../lib/image');
const {pipeline} = require('@tryghost/promise');
const validation = require('../data/validation');
const permissions = require('../services/permissions');
const urlUtils = require('../../shared/url-utils');
const activeStates = ['active', 'warn-1', 'warn-2', 'warn-3', 'warn-4'];
/**
@ -114,6 +115,26 @@ User = ghostBookshelf.Model.extend({
const tasks = [];
let passwordValidation = {};
const urlTransformMap = {
profile_image: 'toTransformReady',
cover_image: 'toTransformReady'
};
Object.entries(urlTransformMap).forEach(([urlAttr, transform]) => {
let method = transform;
let methodOptions = {};
if (typeof transform === 'object') {
method = transform.method;
methodOptions = transform.options || {};
}
if (this.hasChanged(urlAttr) && this.get(urlAttr)) {
const transformedValue = urlUtils[method](this.get(urlAttr), methodOptions);
this.set(urlAttr, transformedValue);
}
});
ghostBookshelf.Model.prototype.onSaving.apply(this, arguments);
/**

6
package.json
View File

@ -51,9 +51,9 @@
"@tryghost/helpers": "1.1.39",
"@tryghost/image-transform": "1.0.3",
"@tryghost/job-manager": "0.8.1",
"@tryghost/kg-card-factory": "2.1.6",
"@tryghost/kg-card-factory": "2.2.0-rc.2",
"@tryghost/kg-default-atoms": "2.0.3",
"@tryghost/kg-default-cards": "4.0.0-rc.6",
"@tryghost/kg-default-cards": "4.0.0-rc.9",
"@tryghost/kg-markdown-html-renderer": "4.0.0-rc.2",
"@tryghost/kg-mobiledoc-html-renderer": "4.0.0-rc.1",
"@tryghost/limit-service": "0.3.0",
@ -67,7 +67,7 @@
"@tryghost/session-service": "0.1.17",
"@tryghost/social-urls": "0.1.19",
"@tryghost/string": "0.1.17",
"@tryghost/url-utils": "1.0.2",
"@tryghost/url-utils": "1.1.0-rc.1",
"@tryghost/vhost-middleware": "1.0.13",
"@tryghost/zip": "1.1.10",
"amperize": "0.6.1",

2
test/regression/api/canary/admin/posts_spec.js
View File

@ -10,7 +10,7 @@ const localUtils = require('./utils');
const ghost = testUtils.startGhost;
let request;
describe('Posts API', function () {
describe('Posts API (canary)', function () {
let ghostServer;
let ownerCookie;

2
test/regression/api/v3/admin/posts_spec.js
View File

@ -10,7 +10,7 @@ const localUtils = require('./utils');
const ghost = testUtils.startGhost;
let request;
describe('Posts API', function () {
describe('Posts API (v3)', function () {
let ghostServer;
let ownerCookie;

22
test/regression/models/model_posts_spec.js
View File

@ -1129,7 +1129,7 @@ describe('Post Model', function () {
it('transforms absolute urls to relative', function (done) {
const post = {
title: 'Absolute->Relative URL Transform Test',
title: 'Absolute->Transform-ready URL Transform Test',
mobiledoc: '{"version":"0.3.1","atoms":[],"cards":[["image",{"src":"http://127.0.0.1:2369/content/images/card.jpg"}]],"markups":[["a",["href","http://127.0.0.1:2369/test"]]],"sections":[[1,"p",[[0,[0],1,"Testing"]]],[10,0]]}',
custom_excerpt: 'Testing <a href="http://127.0.0.1:2369/internal">links</a> in custom excerpts',
codeinjection_head: '<script src="http://127.0.0.1:2369/assets/head.js"></script>',
@ -1143,18 +1143,18 @@ describe('Post Model', function () {
};
models.Post.add(post, context).then((createdPost) => {
createdPost.get('mobiledoc').should.equal('{"version":"0.3.1","atoms":[],"cards":[["image",{"src":"/content/images/card.jpg"}]],"markups":[["a",["href","/test"]]],"sections":[[1,"p",[[0,[0],1,"Testing"]]],[10,0]]}');
createdPost.get('html').should.equal('<p><a href="/test">Testing</a></p><figure class="kg-card kg-image-card"><img src="/content/images/card.jpg" class="kg-image" alt loading="lazy"></figure>');
createdPost.get('custom_excerpt').should.equal('Testing <a href="/internal">links</a> in custom excerpts');
createdPost.get('codeinjection_head').should.equal('<script src="/assets/head.js"></script>');
createdPost.get('codeinjection_foot').should.equal('<script src="/assets/foot.js"></script>');
createdPost.get('feature_image').should.equal('/content/images/feature.png');
createdPost.get('canonical_url').should.equal('/canonical');
createdPost.get('mobiledoc').should.equal('{"version":"0.3.1","atoms":[],"cards":[["image",{"src":"__GHOST_URL__/content/images/card.jpg"}]],"markups":[["a",["href","__GHOST_URL__/test"]]],"sections":[[1,"p",[[0,[0],1,"Testing"]]],[10,0]]}');
createdPost.get('html').should.equal('<p><a href="__GHOST_URL__/test">Testing</a></p><figure class="kg-card kg-image-card"><img src="__GHOST_URL__/content/images/card.jpg" class="kg-image" alt loading="lazy"></figure>');
createdPost.get('custom_excerpt').should.equal('Testing <a href="__GHOST_URL__/internal">links</a> in custom excerpts');
createdPost.get('codeinjection_head').should.equal('<script src="__GHOST_URL__/assets/head.js"></script>');
createdPost.get('codeinjection_foot').should.equal('<script src="__GHOST_URL__/assets/foot.js"></script>');
createdPost.get('feature_image').should.equal('__GHOST_URL__/content/images/feature.png');
createdPost.get('canonical_url').should.equal('__GHOST_URL__/canonical');
const postMeta = createdPost.relations.posts_meta;
postMeta.get('og_image').should.equal('/content/images/og.png');
postMeta.get('twitter_image').should.equal('/content/images/twitter.png');
postMeta.get('og_image').should.equal('__GHOST_URL__/content/images/og.png');
postMeta.get('twitter_image').should.equal('__GHOST_URL__/content/images/twitter.png');
// ensure canonical_url is not transformed when protocol does not match
return createdPost.save({
@ -1164,7 +1164,7 @@ describe('Post Model', function () {
});
}).then((updatedPost) => {
updatedPost.get('canonical_url').should.equal('https://127.0.0.1:2369/https-internal');
updatedPost.get('feature_image').should.equal('/content/images/updated_feature.png');
updatedPost.get('feature_image').should.equal('__GHOST_URL__/content/images/updated_feature.png');
done();
}).catch(done);

18
test/unit/api/canary/utils/serializers/output/utils/url_spec.js
View File

@ -9,9 +9,7 @@ describe('Unit: canary/utils/serializers/output/utils/url', function () {
beforeEach(function () {
sinon.stub(urlService, 'getUrlByResourceId').returns('getUrlByResourceId');
sinon.stub(urlUtils, 'urlFor').returns('urlFor');
sinon.stub(urlUtils, 'relativeToAbsolute').returns('relativeToAbsolute');
sinon.stub(urlUtils, 'htmlRelativeToAbsolute').returns({html: sinon.stub()});
sinon.stub(urlUtils, 'mobiledocRelativeToAbsolute').returns({});
sinon.stub(urlUtils, 'transformReadyToAbsolute').returns('transformReadyToAbsolute');
});
afterEach(function () {
@ -47,18 +45,8 @@ describe('Unit: canary/utils/serializers/output/utils/url', function () {
post.hasOwnProperty('url').should.be.true();
// feature_image, og_image, twitter_image, canonical_url
urlUtils.relativeToAbsolute.callCount.should.eql(4);
// mobiledoc
urlUtils.mobiledocRelativeToAbsolute.callCount.should.eql(1);
// html, codeinjection_head, codeinjection_foot
urlUtils.htmlRelativeToAbsolute.callCount.should.eql(3);
urlUtils.htmlRelativeToAbsolute.getCall(0).args.should.eql([
'html',
'getUrlByResourceId'
]);
// feature_image, og_image, twitter_image, canonical_url, mobiledoc, html, codeinjection_head, codeinjection_foot
urlUtils.transformReadyToAbsolute.callCount.should.eql(8);
urlService.getUrlByResourceId.callCount.should.eql(1);
urlService.getUrlByResourceId.getCall(0).args.should.eql(['id1', {absolute: true}]);

17
test/unit/api/v2/utils/serializers/output/utils/url_spec.js
View File

@ -9,9 +9,7 @@ describe('Unit: v2/utils/serializers/output/utils/url', function () {
beforeEach(function () {
sinon.stub(urlService, 'getUrlByResourceId').returns('getUrlByResourceId');
sinon.stub(urlUtils, 'urlFor').returns('urlFor');
sinon.stub(urlUtils, 'relativeToAbsolute').returns('relativeToAbsolute');
sinon.stub(urlUtils, 'htmlRelativeToAbsolute').returns({html: sinon.stub()});
sinon.stub(urlUtils, 'mobiledocRelativeToAbsolute').returns({});
sinon.stub(urlUtils, 'transformReadyToAbsolute').returns('transformReadyToAbsolute');
});
afterEach(function () {
@ -47,17 +45,12 @@ describe('Unit: v2/utils/serializers/output/utils/url', function () {
post.hasOwnProperty('url').should.be.true();
// feature_image, og_image, twitter_image, canonical_url
urlUtils.relativeToAbsolute.callCount.should.eql(4);
// feature_image, og_image, twitter_image, canonical_url, mobiledoc, html, codeinjection_head, codeinjection_foot
urlUtils.transformReadyToAbsolute.callCount.should.eql(8);
// mobiledoc
urlUtils.mobiledocRelativeToAbsolute.callCount.should.eql(1);
// html, codeinjection_head, codeinjection_foot
urlUtils.htmlRelativeToAbsolute.callCount.should.eql(3);
urlUtils.htmlRelativeToAbsolute.getCall(0).args.should.eql([
// html
urlUtils.transformReadyToAbsolute.getCall(1).args.should.eql([
'html',
'getUrlByResourceId',
{assetsOnly: true}
]);

18
test/unit/api/v3/utils/serializers/output/utils/url_spec.js
View File

@ -9,9 +9,7 @@ describe('Unit: v3/utils/serializers/output/utils/url', function () {
beforeEach(function () {
sinon.stub(urlService, 'getUrlByResourceId').returns('getUrlByResourceId');
sinon.stub(urlUtils, 'urlFor').returns('urlFor');
sinon.stub(urlUtils, 'relativeToAbsolute').returns('relativeToAbsolute');
sinon.stub(urlUtils, 'htmlRelativeToAbsolute').returns({html: sinon.stub()});
sinon.stub(urlUtils, 'mobiledocRelativeToAbsolute').returns({});
sinon.stub(urlUtils, 'transformReadyToAbsolute').returns('transformReadyToAbsolute');
});
afterEach(function () {
@ -47,18 +45,8 @@ describe('Unit: v3/utils/serializers/output/utils/url', function () {
post.hasOwnProperty('url').should.be.true();
// feature_image, og_image, twitter_image, canonical_url
urlUtils.relativeToAbsolute.callCount.should.eql(4);
// mobiledoc
urlUtils.mobiledocRelativeToAbsolute.callCount.should.eql(1);
// html, codeinjection_head, codeinjection_foot
urlUtils.htmlRelativeToAbsolute.callCount.should.eql(3);
urlUtils.htmlRelativeToAbsolute.getCall(0).args.should.eql([
'html',
'getUrlByResourceId'
]);
// feature_image, og_image, twitter_image, canonical_url, mobiledoc, html, codeinjection_head, codeinjection_foot
urlUtils.transformReadyToAbsolute.callCount.should.eql(8);
urlService.getUrlByResourceId.callCount.should.eql(1);
urlService.getUrlByResourceId.getCall(0).args.should.eql(['id1', {absolute: true}]);

15
test/unit/services/rss/generate-feed_spec.js
View File

@ -218,6 +218,11 @@ describe('RSS: Generate Feed', function () {
it('should process urls correctly', function (done) {
data.posts = [posts[3]];
// raw data has __GHOST_URL__ urls but normally the API would have transformed those to absolute
let serializedPosts = JSON.stringify(data.posts);
serializedPosts = serializedPosts.replace(/__GHOST_URL__/g, 'http://my-ghost-blog.com');
data.posts = JSON.parse(serializedPosts);
generateFeed(baseUrl, data).then(function (xmlData) {
should.exist(xmlData);
@ -252,9 +257,15 @@ describe('RSS: Generate Feed', function () {
it('should process urls correctly with subdirectory', function (done) {
baseUrl = '/blog/rss/';
data.results = {posts: [posts[3]], meta: {pagination: {pages: 1}}};
data.posts = [posts[3]];
data.meta = {pagination: {pages: 1}};
generateFeed(baseUrl, data).then(function (xmlData) {
// raw data has __GHOST_URL__ urls but normally the API would have transformed those to absolute
let serializedData = JSON.stringify(data);
serializedData = serializedData.replace(/__GHOST_URL__/g, 'http://my-ghost-blog.com/blog');
const transformedData = JSON.parse(serializedData);
generateFeed(baseUrl, transformedData).then(function (xmlData) {
should.exist(xmlData);
// anchor URL - <a href="#nowhere" title="Anchor URL">

8
test/utils/fixtures/data-generator.js
View File

File diff suppressed because one or more lines are too long

55
yarn.lock
View File

@ -445,10 +445,10 @@
fastq "1.10.1"
p-wait-for "3.2.0"
"@tryghost/kg-card-factory@2.1.6":
version "2.1.6"
resolved "https://registry.yarnpkg.com/@tryghost/kg-card-factory/-/kg-card-factory-2.1.6.tgz#e2f66647773575f941f4b8131b9ce7d0ab60c7a5"
integrity sha512-FmCGXdrvOw8VdJ/rb+7hKbT2R1S4Al8BftUPtPEzduzwzK9BdQb3FohB0o8ofSZy/XvZFVlfRWPCyLAtda9LBA==
"@tryghost/kg-card-factory@2.2.0-rc.2":
version "2.2.0-rc.2"
resolved "https://registry.yarnpkg.com/@tryghost/kg-card-factory/-/kg-card-factory-2.2.0-rc.2.tgz#72b83e9b3a80327136d76b2de735cc4f1a6b63b3"
integrity sha512-JKItId0tMlIbMw4kDrf+iFhaw8hDc6Pmq+54qQfY9K3r3NrezdGtaZPxPuHswJvG0/xPb18LIQNjxbif5R3Wqw==
"@tryghost/kg-clean-basic-html@^1.0.11":
version "1.0.11"
@ -460,17 +460,17 @@
resolved "https://registry.yarnpkg.com/@tryghost/kg-default-atoms/-/kg-default-atoms-2.0.3.tgz#b4a7a4c502a7b9940854cbcf7868b0a4f23b9edd"
integrity sha512-ZC3Lk7X0fGB+nPBSVF3PeirYuEX9sjNd5awmr5X//q8B5UdtUdKqzkW7DvYyABmI0/iL7HkUeZvETx22b3V7bw==
"@tryghost/kg-default-cards@4.0.0-rc.6":
version "4.0.0-rc.6"
resolved "https://registry.yarnpkg.com/@tryghost/kg-default-cards/-/kg-default-cards-4.0.0-rc.6.tgz#896a22d4c5bf660d85786234142d6571973bcffc"
integrity sha512-Ra6qI251y8/2WKV6WEPnDIiy4Abw4zsnryn8V6aRacYTWD2WmAq/ISaiIQok+YntE9F1V00vIMR6bmj1KSy+VQ==
"@tryghost/kg-default-cards@4.0.0-rc.9":
version "4.0.0-rc.9"
resolved "https://registry.yarnpkg.com/@tryghost/kg-default-cards/-/kg-default-cards-4.0.0-rc.9.tgz#187d865fa198af9013d105ae5c4955c1f8059db5"
integrity sha512-AlqvUFwbXxQR9s14/FFhSVjAqa1R7AJwQodjEnQtQdIanV+QlpdboGJM2fJP8CFwVIrE9zwla51G7lh5GOMq+w==
dependencies:
"@tryghost/kg-markdown-html-renderer" "^3.0.1-4.0.0-rc.0.0"
"@tryghost/url-utils" "^0.6.14"
"@tryghost/kg-markdown-html-renderer" "^4.0.0-rc.0"
"@tryghost/url-utils" "^1.1.0-rc.1"
handlebars "^4.7.6"
juice "^7.0.0"
"@tryghost/kg-markdown-html-renderer@4.0.0-rc.2":
"@tryghost/kg-markdown-html-renderer@4.0.0-rc.2", "@tryghost/kg-markdown-html-renderer@^4.0.0-rc.0":
version "4.0.0-rc.2"
resolved "https://registry.yarnpkg.com/@tryghost/kg-markdown-html-renderer/-/kg-markdown-html-renderer-4.0.0-rc.2.tgz#080b96cb5dc3a95151964f133fad760bbfdf8fea"
integrity sha512-OPF8R/lB7akGYNaTT1ooQBwXxqT9y2jrleM65u1/w9qztwSLYfqF6YUazh88YHFVrT7B0rQjQQ3pnHmXeiULzw==
@ -482,17 +482,6 @@
markdown-it-mark "^3.0.0"
semver "^7.3.4"
"@tryghost/kg-markdown-html-renderer@^3.0.1-4.0.0-rc.0.0":
version "3.0.1-4.0.0-rc.0.0"
resolved "https://registry.yarnpkg.com/@tryghost/kg-markdown-html-renderer/-/kg-markdown-html-renderer-3.0.1-4.0.0-rc.0.0.tgz#b87ff60bc12c23445c69075dcfe0c7830d05c59e"
integrity sha512-z9n7dAlr7CouU0Y+p/iBE3RdhAw+rj+htlMsdWGJai2YjefAFxlZ9IHU2vXWwGT+z5S5Pp3hooxF5ZqfYF10Hw==
dependencies:
markdown-it "^12.0.0"
markdown-it-footnote "^3.0.2"
markdown-it-lazy-headers "^0.1.3"
markdown-it-mark "^3.0.0"
semver "^7.3.4"
"@tryghost/kg-mobiledoc-html-renderer@4.0.0-rc.1":
version "4.0.0-rc.1"
resolved "https://registry.yarnpkg.com/@tryghost/kg-mobiledoc-html-renderer/-/kg-mobiledoc-html-renderer-4.0.0-rc.1.tgz#5002a50c9a5f93a4e26de0618421a7839444635c"
@ -631,10 +620,10 @@
dependencies:
unidecode "^0.1.8"
"@tryghost/url-utils@1.0.2":
version "1.0.2"
resolved "https://registry.yarnpkg.com/@tryghost/url-utils/-/url-utils-1.0.2.tgz#05d54dd2f89e254d4379d54d2d877be12ade4635"
integrity sha512-YjUefzev1vvpYBFGxfhO4U6Wew2Swgy3zXz0/Zb0f+FWG0sJ13tplaRcN+79vF0x4IROjxOyn3wR8STyAvQ8JA==
"@tryghost/url-utils@1.1.0-rc.1", "@tryghost/url-utils@^1.1.0-rc.1":
version "1.1.0-rc.1"
resolved "https://registry.yarnpkg.com/@tryghost/url-utils/-/url-utils-1.1.0-rc.1.tgz#796e3518841e5a8e6a83694299ecb147b44aa074"
integrity sha512-QsWgCk0+HdIphcnD9NuoxbYbebiPAZua5zDIAhgitoR6GhvmCERijYqgUW0qppwu/uIgBJsQGYxTTNHSjp/i6Q==
dependencies:
cheerio "0.22.0"
moment "2.27.0"
@ -643,18 +632,6 @@
remark-footnotes "^1.0.0"
unist-util-visit "^2.0.0"
"@tryghost/url-utils@^0.6.14":
version "0.6.18"
resolved "https://registry.yarnpkg.com/@tryghost/url-utils/-/url-utils-0.6.18.tgz#e1c8ab1cbb4f97b2f04a12f6c55b7f41ccf5ca84"
integrity sha512-nYx6qs8gaz1b6Rd9ntv9mQ35EvwR74YnHok1EF22Udm+VfMtQXTaOHviAQqPy4OuEizZqyqj5i8cHEwfH55CKg==
dependencies:
cheerio "0.22.0"
moment "2.24.0"
moment-timezone "0.5.28"
remark "^11.0.2"
remark-footnotes "^1.0.0"
unist-util-visit "^2.0.0"
"@tryghost/vhost-middleware@1.0.13":
version "1.0.13"
resolved "https://registry.yarnpkg.com/@tryghost/vhost-middleware/-/vhost-middleware-1.0.13.tgz#a69234507dd46c27329aa89bade66341265e4c4a"
@ -6623,7 +6600,7 @@ module-not-found-error@^1.0.1:
resolved "https://registry.yarnpkg.com/module-not-found-error/-/module-not-found-error-1.0.1.tgz#cf8b4ff4f29640674d6cdd02b0e3bc523c2bbdc0"
integrity sha1-z4tP9PKWQGdNbN0CsOO8UjwrvcA=
moment-timezone@0.5.23, moment-timezone@0.5.28, moment-timezone@0.5.31:
moment-timezone@0.5.23, moment-timezone@0.5.31:
version "0.5.23"
resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.23.tgz#7cbb00db2c14c71b19303cb47b0fb0a6d8651463"
integrity sha512-WHFH85DkCfiNMDX5D3X7hpNH3/PUhjTGcD0U1SgfBGZxJ3qUmJh5FdvaFjcClxOvB3rzdfj4oRffbI38jEnC1w==