Added a column disallow list in the content API posts serializer (#20207)

ref https://linear.app/tryghost/issue/CFR-29
- Removed the mobiledoc and lexical columns from the posts input
serializer, meaning they will no longer be queried for.

Get helpers are essentially a gateway to the Content API. We already
strip out the mobiledoc and lexical fields in the output
serializer/returned response, but this means we're passing the mobiledoc
and lexical fields back from the db. This is pointless and these fields
are substantial in size - by far the largest fields in the whole ghost
db - leading to slowed performance.

I've updated the posts input serializer to strip out the lexical and mobiledoc
columns so we stop doing a `select *` with every query.

ghost/core/core/server/api/endpoints/utils/serializers/input/posts.js

@@ -7,6 +7,7 @@ const slugFilterOrder = require('./utils/slug-filter-order');
 const localUtils = require('../../index');
 const mobiledoc = require('../../../../../lib/mobiledoc');
 const postsMetaSchema = require('../../../../../data/schema').tables.posts_meta;
+const postsSchema = require('../../../../../data/schema').tables.posts;
 const clean = require('./utils/clean');
 const lexical = require('../../../../../lib/lexical');
 const sentry = require('../../../../../../shared/sentry');
@@ -16,6 +17,16 @@ const messages = {
     failedHtmlToLexical: 'Failed to convert HTML to Lexical'
 };
 
+/**
+ * Selects all allowed columns for the given frame.
+ * 
+ * NOTE: This doesn't stop them from being FETCHED, just returned in the response. This causes
+ *   the output serializer to remove them from the data object before returning.
+ * 
+ * NOTE: This is only intended for the Content API. We need these fields within Admin API responses.
+ *
+ * @param {Object} frame - The frame object.
+ */
 function removeSourceFormats(frame) {
     if (frame.options.formats?.includes('mobiledoc') || frame.options.formats?.includes('lexical')) {
         frame.options.formats = frame.options.formats.filter((format) => {
@@ -24,6 +35,33 @@ function removeSourceFormats(frame) {
     }
 }
 
+/**
+ * Selects all allowed columns for the given frame.
+ * 
+ * This removes the lexical and mobiledoc columns from the query. This is a performance improvement as we never intend
+ *  to expose those columns in the content API and they are very large datasets to be passing around and de/serializing.
+ * 
+ * NOTE: This is only intended for the Content API. We need these fields within Admin API responses.
+ *
+ * @param {Object} frame - The frame object.
+ */
+function selectAllAllowedColumns(frame) {
+    if (!frame.options.columns && !frame.options.selectRaw) {
+        // Because we're returning columns directly from the table we need to remove info columns like @@UNIQUE_CONSTRAINTS@@
+        frame.options.selectRaw = _.keys(_.omit(postsSchema, ['lexical','mobiledoc','@@UNIQUE_CONSTRAINTS@@'])).join(',');
+    } else if (frame.options.columns) {
+        frame.options.columns = frame.options.columns.filter((column) => {
+            return !['mobiledoc', 'lexical'].includes(column);
+        });
+    } else if (frame.options.selectRaw) {
+        frame.options.selectRaw = frame.options.selectRaw.split(',').map((column) => {
+            return column.trim();
+        }).filter((column) => {
+            return !['mobiledoc', 'lexical'].includes(column);
+        }).join(',');
+    }
+}
+
 /**
  * Map names of relations to the internal names
  */
@@ -128,7 +166,8 @@ module.exports = {
          */
         if (localUtils.isContentAPI(frame)) {
             // CASE: the content api endpoint for posts should not return mobiledoc or lexical
-            removeSourceFormats(frame);
+            removeSourceFormats(frame); // remove from the format field
+            selectAllAllowedColumns(frame); // remove from any specified column or selectRaw options
 
             setDefaultOrder(frame);
             forceVisibilityColumn(frame);

ghost/core/test/unit/api/canary/utils/serializers/input/posts.test.js

@@ -1,6 +1,7 @@
 const should = require('should');
 const sinon = require('sinon');
 const serializers = require('../../../../../../../core/server/api/endpoints/utils/serializers');
+const postsSchema = require('../../../../../../../core/server/data/schema').tables.posts;
 
 const mobiledocLib = require('@tryghost/html-to-mobiledoc');
 
@@ -100,6 +101,65 @@ describe('Unit: endpoints/utils/serializers/input/posts', function () {
             frame.options.formats.should.containEql('html');
             frame.options.formats.should.containEql('plaintext');
         });
+
+        describe('Content API', function () {
+            it('selects all columns from the posts schema but mobiledoc and lexical when no columns are specified', function () {
+                const apiConfig = {};
+                const frame = {
+                    apiType: 'content',
+                    options: {
+                        context: {}
+                    }
+                };
+
+                serializers.input.posts.browse(apiConfig, frame);
+                const columns = Object.keys(postsSchema);
+                const parsedSelectRaw = frame.options.selectRaw.split(',').map(column => column.trim());
+                parsedSelectRaw.should.eql(columns.filter(column => !['mobiledoc', 'lexical','@@UNIQUE_CONSTRAINTS@@'].includes(column)));
+            });
+
+            it('strips mobiledoc and lexical columns from a specified columns option', function () {
+                const apiConfig = {};
+                const frame = {
+                    apiType: 'content',
+                    options: {
+                        context: {},
+                        columns: ['id', 'mobiledoc', 'lexical', 'visibility']
+                    }
+                };
+
+                serializers.input.posts.browse(apiConfig, frame);
+                frame.options.columns.should.eql(['id', 'visibility']);
+            });
+
+            it('forces visibility column if columns are specified', function () {
+                const apiConfig = {};
+                const frame = {
+                    apiType: 'content',
+                    options: {
+                        context: {},
+                        columns: ['id']
+                    }
+                };
+                
+                serializers.input.posts.browse(apiConfig, frame);
+                frame.options.columns.should.eql(['id', 'visibility']);
+            });
+                
+            it('strips mobiledoc and lexical columns from a specified selectRaw option', function () {
+                const apiConfig = {};
+                const frame = {
+                    apiType: 'content',
+                    options: {
+                        context: {},
+                        selectRaw: 'id, mobiledoc, lexical'
+                    }
+                };
+
+                serializers.input.posts.browse(apiConfig, frame);
+                frame.options.selectRaw.should.eql('id');
+            });
+        });
     });
 
     describe('read', function () {