Added mobiledoc transform to populate missing image sizes on forced re-render (#11924)

no issue

- adds `populateImageSizes()` to our mobiledoc lib module
  - uses `image-size` lib to speed up reading of image dimensions
  - for local images, use storage adapter with same guards as used by `handle-image-sizes` middleware so that we don't insert srcsets for images that aren't transformable
  - for unsplash images, remove any width and crop params from the url so it points to the full-size image
- use `populateImageSizes(mobiledoc)` to modify post model's mobiledoc when re-rendering

core/server/lib/mobiledoc.js

@@ -1,6 +1,9 @@
+const path = require('path');
 const errors = require('@tryghost/errors');
+const imageTransform = require('@tryghost/image-transform');
 const logging = require('../../shared/logging');
 const config = require('../../shared/config');
+const storage = require('../adapters/storage');
 
 let cardFactory;
 let cards;
@@ -78,6 +81,83 @@ module.exports = {
         }
     },
 
+    // used when force-rerendering post content to ensure that old image card
+    // payloads contain width/height values to be used when generating srcsets
+    populateImageSizes: async function (mobiledocJson) {
+        // do not require image-size until it's requested to avoid circular dependencies
+        // shared/url-utils > server/lib/mobiledoc > server/lib/image/image-size > server/adapters/storage/utils
+        const imageSize = require('./image/image-size');
+        const urlUtils = require('../../shared/url-utils');
+        const storageInstance = storage.getStorage();
+
+        async function getUnsplashSize(url) {
+            const parsedUrl = new URL(url);
+            parsedUrl.searchParams.delete('w');
+            parsedUrl.searchParams.delete('fit');
+            parsedUrl.searchParams.delete('crop');
+            parsedUrl.searchParams.delete('dpr');
+
+            return await imageSize.getImageSizeFromUrl(parsedUrl.href);
+        }
+
+        // TODO: extract conditional logic lifted from handle-image-sizes.js
+        async function getLocalSize(url) {
+            // skip local images if adapter doesn't support size transforms
+            if (typeof storageInstance.saveRaw !== 'function') {
+                return;
+            }
+
+            // local storage adapter's .exists() expects image paths without any prefixes
+            const imageUrlPrefix = urlUtils.urlJoin(urlUtils.getSubdir(), urlUtils.STATIC_IMAGE_URL_PREFIX);
+            const storagePath = url.replace(imageUrlPrefix, '');
+
+            const {dir, name, ext} = path.parse(storagePath);
+            const [imageNameMatched, imageName, imageNumber] = name.match(/^(.+?)(-\d+)?$/) || [null];
+
+            if (!imageNameMatched
+                || !imageTransform.canTransformFileExtension(ext)
+                || !(await storageInstance.exists(storagePath))
+            ) {
+                return;
+            }
+
+            // get the original/unoptimized image if it exists as that will have
+            // the maximum dimensions that srcset/handle-image-sizes can use
+            const originalImagePath = path.join(dir, `${imageName}_o${imageNumber || ''}${ext}`);
+            const imagePath = await storageInstance.exists(originalImagePath) ? originalImagePath : storagePath;
+
+            return await imageSize.getImageSizeFromStoragePath(imagePath);
+        }
+
+        const mobiledoc = JSON.parse(mobiledocJson);
+
+        const sizePromises = mobiledoc.cards.map(async (card) => {
+            const [cardName, payload] = card;
+
+            const needsFilling = cardName === 'image' && (!payload.width || !payload.height);
+            if (!needsFilling) {
+                return;
+            }
+
+            const isUnsplash = payload.src.match(/images\.unsplash\.com/);
+            try {
+                const size = isUnsplash ? await getUnsplashSize(payload.src) : await getLocalSize(payload.src);
+
+                if (size && size.width && size.height) {
+                    payload.width = size.width;
+                    payload.height = size.height;
+                }
+            } catch (e) {
+                // TODO: use debug instead?
+                logging.error(e);
+            }
+        });
+
+        await Promise.all(sizePromises);
+
+        return JSON.stringify(mobiledoc);
+    },
+
     // allow config changes to be picked up - useful in tests
     reload() {
         cardFactory = null;

core/server/models/post.js

@@ -404,6 +404,12 @@ Post = ghostBookshelf.Model.extend({
             }
         });
 
+        // If we're force re-rendering we want to make sure that all image cards
+        // have original dimensions stored in the payload for use by card renderers
+        if (options.force_rerender) {
+            this.set('mobiledoc', mobiledocLib.populateImageDimensions(this.get('mobiledoc')));
+        }
+
         // CASE: mobiledoc has changed, generate html
         // CASE: ?force_rerender=true passed via Admin API
         // CASE: html is null, but mobiledoc exists (only important for migrations & importing)

core/shared/url-utils.js

@@ -1,6 +1,5 @@
 const UrlUtils = require('@tryghost/url-utils');
 const config = require('./config');
-const mobiledoc = require('../server/lib/mobiledoc');
 
 const urlUtils = new UrlUtils({
     url: config.get('url'),
@@ -11,6 +10,9 @@ const urlUtils = new UrlUtils({
     redirectCacheMaxAge: config.get('caching:301:maxAge'),
     baseApiPath: '/ghost/api',
     get cardTransformers() {
+        // do not require mobiledoc until it's requested to avoid circular dependencies
+        // shared/url-utils > server/lib/mobiledoc > server/lib/image/image-size > server/adapters/storage/utils
+        const mobiledoc = require('../server/lib/mobiledoc');
         return mobiledoc.cards;
     }
 });

test/unit/lib/mobiledoc_spec.js

@@ -1,6 +1,9 @@
+const path = require('path');
 const should = require('should');
+const nock = require('nock');
 const configUtils = require('../../utils/configUtils');
 const mobiledocLib = require('../../../core/server/lib/mobiledoc');
+const storage = require('../../../core/server/adapters/storage');
 
 describe('lib/mobiledoc', function () {
     beforeEach(function () {
@@ -8,6 +11,7 @@ describe('lib/mobiledoc', function () {
     });
 
     afterEach(function () {
+        nock.cleanAll();
         configUtils.restore();
         // ensure config changes are reset and picked up by next test
         mobiledocLib.reload();
@@ -110,4 +114,54 @@ describe('lib/mobiledoc', function () {
                 .should.eql('<figure class="kg-card kg-image-card kg-width-wide kg-card-hascaption"><img src="/content/images/2018/04/NatGeo06.jpg" class="kg-image" alt><figcaption>Birdies</figcaption></figure><figure class="kg-card kg-gallery-card kg-width-wide"><div class="kg-gallery-container"><div class="kg-gallery-row"><div class="kg-gallery-image"><img src="/content/images/test.png" width="1000" height="500" alt></div></div></div></figure>');
         });
     });
+
+    describe('populateImageSizes', function () {
+        let originalStoragePath;
+
+        beforeEach(function () {
+            originalStoragePath = storage.getStorage().storagePath;
+            storage.getStorage().storagePath = path.join(__dirname, '../../utils/fixtures/images/');
+        });
+
+        afterEach(function () {
+            storage.getStorage().storagePath = originalStoragePath;
+        });
+
+        it('works', async function () {
+            let mobiledoc = {
+                cards: [
+                    ['image', {src: '/content/images/ghost-logo.png'}],
+                    ['image', {src: 'http://example.com/external.jpg'}],
+                    ['image', {src: 'https://images.unsplash.com/favicon_too_large?ixlib=rb-1.2.1&q=80&fm=jpg&crop=entropy&cs=tinysrgb&w=2000&fit=max&ixid=eyJhcHBfaWQiOjExNzczfQ'}]
+                ]
+            };
+
+            const unsplashMock = nock('https://images.unsplash.com/')
+                .get('/favicon_too_large')
+                .query(true)
+                .replyWithFile(200, path.join(__dirname, '../../utils/fixtures/images/favicon_not_square.png'), {
+                    'Content-Type': 'image/png'
+                });
+
+            const transformedMobiledoc = await mobiledocLib.populateImageSizes(JSON.stringify(mobiledoc));
+            const transformed = JSON.parse(transformedMobiledoc);
+
+            unsplashMock.isDone().should.be.true();
+
+            transformed.cards.length.should.equal(3);
+
+            should.exist(transformed.cards[0][1].width);
+            transformed.cards[0][1].width.should.equal(800);
+            should.exist(transformed.cards[0][1].height);
+            transformed.cards[0][1].height.should.equal(257);
+
+            should.not.exist(transformed.cards[1][1].width);
+            should.not.exist(transformed.cards[1][1].height);
+
+            should.exist(transformed.cards[2][1].width);
+            transformed.cards[2][1].width.should.equal(100);
+            should.exist(transformed.cards[2][1].height);
+            transformed.cards[2][1].height.should.equal(80);
+        });
+    });
 });