Moved serialisation of formats into the serialiser-layer

This prepares us to return a DTO rather than BookshelfModel to the serialiser
layer. When passing a BookshelfModel, the serialisation layer uses the model to
read from when building computed properties. By stripping values out in the
toJSON method it means that the DTO will be missing them and the computed
properties won't be able to be calculated. Instead we return ALL values to the
serialisation layer, and then strip out the ones that weren't requested in the
"clean" step.

This also inadvertently fixes the issue with `reading_time` requiring the
`html` field to be requested, we can now request just `reading_time`, as well
as have it included by default.

ghost/core/core/server/api/endpoints/utils/serializers/output/mappers/posts.js

@@ -46,6 +46,16 @@ module.exports = async (model, frame, options = {}) => {
 
     extraAttrs.forPost(frame.options, model, jsonModel);
 
+    const defaultFormats = ['html'];
+    const formatsToKeep = frame.options.formats || frame.options.columns || defaultFormats;
+
+    // Iterate over all known formats, and if they are not in the keep list, remove them
+    _.each(['mobiledoc', 'lexical', 'html', 'plaintext'], function (format) {
+        if (formatsToKeep.indexOf(format) === -1) {
+            delete jsonModel[format];
+        }
+    });
+
     // Attach tiers to custom nql visibility filter
     if (jsonModel.visibility) {
         if (['members', 'public'].includes(jsonModel.visibility) && jsonModel.tiers) {

ghost/core/core/server/api/endpoints/utils/serializers/output/utils/clean.js

@@ -78,6 +78,7 @@ const author = (attrs, frame) => {
 const post = (attrs, frame) => {
     const columns = frame && frame.options && frame.options.columns || null;
     const fields = frame && frame.original && frame.original.query && frame.original.query.fields || null;
+
     if (localUtils.isContentAPI(frame)) {
         delete attrs.status;
         delete attrs.email_only;

ghost/core/core/server/models/post.js

@@ -997,6 +997,9 @@ Post = ghostBookshelf.Model.extend({
     /**
      * If the `formats` option is not used, we return `html` be default.
      * Otherwise we return what is requested e.g. `?formats=mobiledoc,plaintext`
+     *
+     * This method is only used by the raw-knex plugin.
+     * We have moved the logic into the serializers for the API.
      */
     formatsToJSON: function formatsToJSON(attrs, options) {
         const defaultFormats = ['html'];
@@ -1016,8 +1019,6 @@ Post = ghostBookshelf.Model.extend({
         const options = Post.filterOptions(unfilteredOptions, 'toJSON');
         let attrs = ghostBookshelf.Model.prototype.toJSON.call(this, options);
 
-        attrs = this.formatsToJSON(attrs, options);
-
         // CASE: never expose the mobiledoc revisions
         delete attrs.mobiledoc_revisions;
 

ghost/core/test/e2e-api/admin/__snapshots__/posts.test.js.snap

@@ -49,6 +49,7 @@ Object {
       "primary_author": Any<Object>,
       "primary_tag": Any<Object>,
       "published_at": StringMatching /\\\\d\\{4\\}-\\\\d\\{2\\}-\\\\d\\{2\\}T\\\\d\\{2\\}:\\\\d\\{2\\}:\\\\d\\{2\\}\\\\\\.000Z/,
+      "reading_time": 0,
       "slug": "scheduled-post",
       "status": "scheduled",
       "tags": Any<Array>,
@@ -137,6 +138,7 @@ Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac tu
       "primary_author": Any<Object>,
       "primary_tag": Any<Object>,
       "published_at": StringMatching /\\\\d\\{4\\}-\\\\d\\{2\\}-\\\\d\\{2\\}T\\\\d\\{2\\}:\\\\d\\{2\\}:\\\\d\\{2\\}\\\\\\.000Z/,
+      "reading_time": 1,
       "slug": "unfinished",
       "status": "draft",
       "tags": Any<Array>,
@@ -195,7 +197,7 @@ exports[`Posts API Can browse 2: [headers] 1`] = `
 Object {
   "access-control-allow-origin": "http://127.0.0.1:2369",
   "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
-  "content-length": "10456",
+  "content-length": "10490",
   "content-type": "application/json; charset=utf-8",
   "content-version": StringMatching /v\\\\d\\+\\\\\\.\\\\d\\+/,
   "etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,

ghost/core/test/e2e-api/admin/pages-legacy.test.js

@@ -29,7 +29,7 @@ describe('Pages API', function () {
         localUtils.API.checkResponse(jsonResponse, 'pages');
         jsonResponse.pages.should.have.length(6);
 
-        localUtils.API.checkResponse(jsonResponse.pages[0], 'page');
+        localUtils.API.checkResponse(jsonResponse.pages[0], 'page', ['reading_time']);
         localUtils.API.checkResponse(jsonResponse.meta.pagination, 'pagination');
         _.isBoolean(jsonResponse.pages[0].featured).should.eql(true);
 
@@ -51,7 +51,7 @@ describe('Pages API', function () {
         localUtils.API.checkResponse(jsonResponse, 'pages');
         jsonResponse.pages.should.have.length(6);
 
-        const additionalProperties = ['lexical'];
+        const additionalProperties = ['lexical', 'reading_time'];
         const missingProperties = ['mobiledoc'];
         localUtils.API.checkResponse(jsonResponse.pages[0], 'page', additionalProperties, missingProperties);
     });
@@ -121,7 +121,7 @@ describe('Pages API', function () {
         res.body.pages.length.should.eql(1);
         const [returnedPage] = res.body.pages;
 
-        const additionalProperties = ['lexical'];
+        const additionalProperties = ['lexical', 'reading_time'];
         localUtils.API.checkResponse(returnedPage, 'page', additionalProperties);
 
         should.equal(returnedPage.mobiledoc, page.mobiledoc);
@@ -346,7 +346,7 @@ describe('Pages API', function () {
             .expect(200);
 
         should.exist(res2.headers['x-cache-invalidate']);
-        localUtils.API.checkResponse(res2.body.pages[0], 'page');
+        localUtils.API.checkResponse(res2.body.pages[0], 'page', ['reading_time']);
 
         const model = await models.Post.findOne({
             id: res2.body.pages[0].id
@@ -389,7 +389,7 @@ describe('Pages API', function () {
             .expect(200);
 
         should.exist(res2.headers['x-cache-invalidate']);
-        localUtils.API.checkResponse(res2.body.pages[0], 'page');
+        localUtils.API.checkResponse(res2.body.pages[0], 'page', ['reading_time']);
         res2.body.pages[0].tiers.length.should.eql(paidTiers.length);
 
         const model = await models.Post.findOne({

ghost/core/test/e2e-api/admin/utils.js

@@ -89,7 +89,8 @@ const expectedProperties = {
         'tiers',
         'newsletter',
         'count',
-        'post_revisions'
+        'post_revisions',
+        'reading_time'
     ],
 
     page: [

ghost/core/test/e2e-api/content/__snapshots__/pages.test.js.snap

@@ -324,6 +324,7 @@ Tip: If you're reading any post or page on your site and you notice something yo
       "og_image": null,
       "og_title": null,
       "published_at": StringMatching /\\\\d\\{4\\}-\\\\d\\{2\\}-\\\\d\\{2\\}T\\\\d\\{2\\}:\\\\d\\{2\\}:\\\\d\\{2\\}\\\\\\.000\\\\\\+\\\\d\\{2\\}:\\\\d\\{2\\}/,
+      "reading_time": 0,
       "slug": "about",
       "title": "About this site",
       "twitter_description": null,
@@ -366,6 +367,7 @@ If you prefer to use a contact form, almost all of the great embedded form servi
       "og_image": null,
       "og_title": null,
       "published_at": StringMatching /\\\\d\\{4\\}-\\\\d\\{2\\}-\\\\d\\{2\\}T\\\\d\\{2\\}:\\\\d\\{2\\}:\\\\d\\{2\\}\\\\\\.000\\\\\\+\\\\d\\{2\\}:\\\\d\\{2\\}/,
+      "reading_time": 1,
       "slug": "contact",
       "title": "Contact",
       "twitter_description": null,
@@ -404,6 +406,7 @@ Ghost is a non-profit organization, and we give away all our intellectual proper
       "og_image": null,
       "og_title": null,
       "published_at": StringMatching /\\\\d\\{4\\}-\\\\d\\{2\\}-\\\\d\\{2\\}T\\\\d\\{2\\}:\\\\d\\{2\\}:\\\\d\\{2\\}\\\\\\.000\\\\\\+\\\\d\\{2\\}:\\\\d\\{2\\}/,
+      "reading_time": 0,
       "slug": "contribute",
       "title": "Contribute",
       "twitter_description": null,
@@ -439,6 +442,7 @@ You can integrate any products, services, ads or integrations with Ghost yoursel
       "og_image": null,
       "og_title": null,
       "published_at": StringMatching /\\\\d\\{4\\}-\\\\d\\{2\\}-\\\\d\\{2\\}T\\\\d\\{2\\}:\\\\d\\{2\\}:\\\\d\\{2\\}\\\\\\.000\\\\\\+\\\\d\\{2\\}:\\\\d\\{2\\}/,
+      "reading_time": 0,
       "slug": "privacy",
       "title": "Privacy",
       "twitter_description": null,
@@ -474,6 +478,7 @@ Hopefully you don't find it a bore.",
       "og_image": null,
       "og_title": null,
       "published_at": StringMatching /\\\\d\\{4\\}-\\\\d\\{2\\}-\\\\d\\{2\\}T\\\\d\\{2\\}:\\\\d\\{2\\}:\\\\d\\{2\\}\\\\\\.000\\\\\\+\\\\d\\{2\\}:\\\\d\\{2\\}/,
+      "reading_time": 0,
       "slug": "static-page-test",
       "title": "This is a static page",
       "twitter_description": null,

ghost/core/test/e2e-api/content/__snapshots__/posts.test.js.snap

@@ -3877,6 +3877,7 @@ Object {
       "og_image": null,
       "og_title": null,
       "published_at": StringMatching /\\\\d\\{4\\}-\\\\d\\{2\\}-\\\\d\\{2\\}T\\\\d\\{2\\}:\\\\d\\{2\\}:\\\\d\\{2\\}\\\\\\.000\\\\\\+\\\\d\\{2\\}:\\\\d\\{2\\}/,
+      "reading_time": 2,
       "slug": "welcome",
       "title": "Start here for a quick overview of everything you need to know",
       "twitter_description": null,
@@ -3911,6 +3912,7 @@ Object {
       "og_image": null,
       "og_title": null,
       "published_at": StringMatching /\\\\d\\{4\\}-\\\\d\\{2\\}-\\\\d\\{2\\}T\\\\d\\{2\\}:\\\\d\\{2\\}:\\\\d\\{2\\}\\\\\\.000\\\\\\+\\\\d\\{2\\}:\\\\d\\{2\\}/,
+      "reading_time": 3,
       "slug": "design",
       "title": "Customizing your brand and design settings",
       "twitter_description": null,
@@ -3945,6 +3947,7 @@ Object {
       "og_image": null,
       "og_title": null,
       "published_at": StringMatching /\\\\d\\{4\\}-\\\\d\\{2\\}-\\\\d\\{2\\}T\\\\d\\{2\\}:\\\\d\\{2\\}:\\\\d\\{2\\}\\\\\\.000\\\\\\+\\\\d\\{2\\}:\\\\d\\{2\\}/,
+      "reading_time": 5,
       "slug": "write",
       "title": "Writing and managing content in Ghost, an advanced guide",
       "twitter_description": null,
@@ -3979,6 +3982,7 @@ Object {
       "og_image": null,
       "og_title": null,
       "published_at": StringMatching /\\\\d\\{4\\}-\\\\d\\{2\\}-\\\\d\\{2\\}T\\\\d\\{2\\}:\\\\d\\{2\\}:\\\\d\\{2\\}\\\\\\.000\\\\\\+\\\\d\\{2\\}:\\\\d\\{2\\}/,
+      "reading_time": 2,
       "slug": "portal",
       "title": "Building your audience with subscriber signups",
       "twitter_description": null,
@@ -4013,6 +4017,7 @@ Object {
       "og_image": null,
       "og_title": null,
       "published_at": StringMatching /\\\\d\\{4\\}-\\\\d\\{2\\}-\\\\d\\{2\\}T\\\\d\\{2\\}:\\\\d\\{2\\}:\\\\d\\{2\\}\\\\\\.000\\\\\\+\\\\d\\{2\\}:\\\\d\\{2\\}/,
+      "reading_time": 1,
       "slug": "sell",
       "title": "Selling premium memberships with recurring revenue",
       "twitter_description": null,
@@ -4047,6 +4052,7 @@ Object {
       "og_image": null,
       "og_title": null,
       "published_at": StringMatching /\\\\d\\{4\\}-\\\\d\\{2\\}-\\\\d\\{2\\}T\\\\d\\{2\\}:\\\\d\\{2\\}:\\\\d\\{2\\}\\\\\\.000\\\\\\+\\\\d\\{2\\}:\\\\d\\{2\\}/,
+      "reading_time": 2,
       "slug": "grow",
       "title": "How to grow your business around an audience",
       "twitter_description": null,
@@ -4081,6 +4087,7 @@ Object {
       "og_image": null,
       "og_title": null,
       "published_at": StringMatching /\\\\d\\{4\\}-\\\\d\\{2\\}-\\\\d\\{2\\}T\\\\d\\{2\\}:\\\\d\\{2\\}:\\\\d\\{2\\}\\\\\\.000\\\\\\+\\\\d\\{2\\}:\\\\d\\{2\\}/,
+      "reading_time": 1,
       "slug": "integrations",
       "title": "Setting up apps and custom integrations",
       "twitter_description": null,
@@ -4126,6 +4133,7 @@ Definition listConsectetur adipisicing elit, sed do eiusmod tempor incididunt ut
       "og_image": null,
       "og_title": null,
       "published_at": StringMatching /\\\\d\\{4\\}-\\\\d\\{2\\}-\\\\d\\{2\\}T\\\\d\\{2\\}:\\\\d\\{2\\}:\\\\d\\{2\\}\\\\\\.000\\\\\\+\\\\d\\{2\\}:\\\\d\\{2\\}/,
+      "reading_time": 1,
       "slug": "not-so-short-bit-complex",
       "title": "Not so short, bit complex",
       "twitter_description": null,
@@ -4169,6 +4177,7 @@ mctesters
       "og_image": null,
       "og_title": null,
       "published_at": StringMatching /\\\\d\\{4\\}-\\\\d\\{2\\}-\\\\d\\{2\\}T\\\\d\\{2\\}:\\\\d\\{2\\}:\\\\d\\{2\\}\\\\\\.000\\\\\\+\\\\d\\{2\\}:\\\\d\\{2\\}/,
+      "reading_time": 0,
       "slug": "short-and-sweet",
       "title": "Short and Sweet",
       "twitter_description": null,
@@ -4205,6 +4214,7 @@ Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac tu
       "og_image": null,
       "og_title": null,
       "published_at": StringMatching /\\\\d\\{4\\}-\\\\d\\{2\\}-\\\\d\\{2\\}T\\\\d\\{2\\}:\\\\d\\{2\\}:\\\\d\\{2\\}\\\\\\.000\\\\\\+\\\\d\\{2\\}:\\\\d\\{2\\}/,
+      "reading_time": 1,
       "slug": "ghostly-kitchen-sink",
       "title": "Ghostly Kitchen Sink",
       "twitter_description": null,
@@ -4239,6 +4249,7 @@ Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac tu
       "og_image": null,
       "og_title": null,
       "published_at": StringMatching /\\\\d\\{4\\}-\\\\d\\{2\\}-\\\\d\\{2\\}T\\\\d\\{2\\}:\\\\d\\{2\\}:\\\\d\\{2\\}\\\\\\.000\\\\\\+\\\\d\\{2\\}:\\\\d\\{2\\}/,
+      "reading_time": 1,
       "slug": "html-ipsum",
       "title": "HTML Ipsum",
       "twitter_description": null,

ghost/core/test/regression/api/admin/utils.js

@@ -67,7 +67,8 @@ const expectedProperties = {
         'tiers',
         'newsletter',
         'count',
-        'post_revisions'
+        'post_revisions',
+        'reading_time'
     ],
     user: [
         'id',

ghost/members-api/lib/repositories/EventRepository.js

@@ -333,10 +333,14 @@ module.exports = class EventRepository {
         const {data: models, meta} = await this._MemberCreatedEvent.findPage(options);
 
         const data = models.map((model) => {
+            const json = model.toJSON(options);
+            delete json.postAttribution?.mobiledoc;
+            delete json.postAttribution?.lexical;
+            delete json.postAttribution?.plaintext;
             return {
                 type: 'signup_event',
                 data: {
-                    ...model.toJSON(options),
+                    ...json,
                     attribution: this._memberAttributionService.getEventAttribution(model)
                 }
             };