Added mailgun provider for sending emails (#15896)

closes https://github.com/TryGhost/Team/issues/2309

- adds new mailgun provider to send out batch emails
- updates sending service to send email id for mailgun provider, allows tagging mail with email id

ghost/core/core/server/services/email-service/wrapper.js

@@ -1,5 +1,4 @@
 const logging = require('@tryghost/logging');
-const ObjectID = require('bson-objectid').default;
 const url = require('../../../server/api/endpoints/utils/serializers/output/utils/url');
 
 class EmailServiceWrapper {
@@ -14,13 +13,16 @@ class EmailServiceWrapper {
             return;
         }
 
-        const {EmailService, EmailController, EmailRenderer, SendingService, BatchSendingService, EmailSegmenter, EmailEventStorage} = require('@tryghost/email-service');
+        const {EmailService, EmailController, EmailRenderer, SendingService, BatchSendingService, EmailSegmenter, EmailEventStorage, MailgunEmailProvider} = require('@tryghost/email-service');
         const {Post, Newsletter, Email, EmailBatch, EmailRecipient, Member} = require('../../models');
+        const MailgunClient = require('@tryghost/mailgun-client');
+        const configService = require('../../../shared/config');
         const settingsCache = require('../../../shared/settings-cache');
         const settingsHelpers = require('../../services/settings-helpers');
         const jobsService = require('../jobs');
         const membersService = require('../members');
         const db = require('../../data/db');
+        const sentry = require('../../../shared/sentry');
         const membersRepository = membersService.api.members;
         const limitService = require('../limits');
         const domainEvents = require('@tryghost/domain-events');
@@ -33,6 +35,22 @@ class EmailServiceWrapper {
         const linkTracking = require('../link-tracking');
         const audienceFeedback = require('../audience-feedback');
 
+        // capture errors from mailgun client and log them in sentry
+        const errorHandler = (error) => {
+            logging.info(`Capturing error for mailgun email provider service`);
+            sentry.captureException(error);
+        };
+
+        // Mailgun client instance for email provider
+        const mailgunClient = new MailgunClient({
+            config: configService, settings: settingsCache
+        });
+
+        const mailgunEmailProvider = new MailgunEmailProvider({
+            mailgunClient,
+            errorHandler
+        });
+
         const emailRenderer = new EmailRenderer({
             settingsCache,
             settingsHelpers,
@@ -50,31 +68,7 @@ class EmailServiceWrapper {
         });
 
         const sendingService = new SendingService({
-            emailProvider: {
+            emailProvider: mailgunEmailProvider,
-                send: async ({plaintext, subject, from, replyTo, recipients}) => {
-                    logging.info(`Sending email\nSubject: ${subject}\nFrom: ${from}\nReplyTo: ${replyTo}\nRecipients: ${recipients.length}\n\n${plaintext}`);
-
-                    // Uncomment to test email HTML rendering with GhostMailer
-                    /*const {GhostMailer} = require('../mail');
-                    const mailer = new GhostMailer();
-                    logging.info(`Sending email\nSubject: ${subject}\nFrom: ${from}\nReplyTo: ${replyTo}\nRecipients: ${recipients.length}\n\n${JSON.stringify(recipients[0].replacements, undefined, '    ')}`);
-                    
-                    for (const replacement of recipients[0].replacements) {
-                        html = html.replace(replacement.token, replacement.value);
-                        plaintext = plaintext.replace(replacement.token, replacement.value);
-                    }
-
-                    await mailer.send({
-                        subject,
-                        html,
-                        to: recipients[0].email,
-                        from,
-                        replyTo,
-                        text: plaintext
-                    });*/
-                    return Promise.resolve({id: 'fake_provider_id_' + ObjectID().toHexString()});
-                }
-            },
             emailRenderer
         });
 

ghost/email-service/index.js

@@ -6,5 +6,6 @@ module.exports = {
     SendingService: require('./lib/sending-service'),
     BatchSendingService: require('./lib/batch-sending-service'),
     EmailEventProcessor: require('./lib/email-event-processor'),
-    EmailEventStorage: require('./lib/email-event-storage')
+    EmailEventStorage: require('./lib/email-event-storage'),
+    MailgunEmailProvider: require('./lib/mailgun-email-provider')
 };

ghost/email-service/lib/batch-sending-service.js

@@ -286,6 +286,7 @@ class BatchSendingService {
         try {
             const members = await this.getBatchMembers(batch.id);
             const response = await this.#sendingService.send({
+                emailId: email.id,
                 post,
                 newsletter,
                 segment: batch.get('member_segment'),
@@ -297,18 +298,21 @@ class BatchSendingService {
 
             await batch.save({
                 status: 'submitted',
-                provider_id: response.id
+                provider_id: response.id,
+                // reset error fields when sending succeeds
+                error_status_code: null,
+                error_message: null,
+                error_data: null
             }, {patch: true, require: false});
             succeeded = true;
         } catch (err) {
             logging.error(`Error sending email batch ${batch.id}`, err);
 
             await batch.save({
-                status: 'failed'
+                status: 'failed',
-                // TODO: error should be instance of EmailProviderError (see IEmailProviderService) + we should read error message
+                error_status_code: err.statusCode,
-                // error_status_code: err.status_code,
+                error_message: err.message,
-                // error_message: err.message_short,
+                error_data: err.errorDetails
-                // error_data: err.message_full
             }, {patch: true, require: false});
         }
 

ghost/email-service/lib/email-renderer.js

@@ -351,7 +351,7 @@ class EmailRenderer {
             {
                 id: 'first_name',
                 getValue: (member) => {
-                    return member.name.split(' ')[0];
+                    return member.name?.split(' ')[0];
                 }
             }
         ];

ghost/email-service/lib/email-service.js

@@ -11,6 +11,7 @@ const errors = require('@tryghost/errors');
 const tpl = require('@tryghost/tpl');
 const EmailRenderer = require('./email-renderer');
 const EmailSegmenter = require('./email-segmenter');
+const MailgunEmailProvider = require('./mailgun-email-provider');
 
 const messages = {
     archivedNewsletterError: 'Cannot send email to archived newsletters',

ghost/email-service/lib/mailgun-email-provider.js

@@ -0,0 +1,168 @@
+const logging = require('@tryghost/logging');
+const errors = require('@tryghost/errors');
+const debug = require('@tryghost/debug')('email-service:mailgun-provider-service');
+
+/**
+ * @typedef {object} Recipient
+ * @prop {string} email
+ * @prop {Replacement[]} replacements
+ */
+
+/**
+ * @typedef {object} Replacement
+ * @prop {string} token
+ * @prop {string} value
+ * @prop {string} id
+ */
+
+/**
+ * @typedef {object} EmailSendingOptions
+ * @prop {boolean} clickTrackingEnabled
+ * @prop {boolean} openTrackingEnabled
+ */
+
+/**
+ * @typedef {object} EmailProviderSuccessResponse
+ * @prop {string} id
+ */
+
+class MailgunEmailProvider {
+    #mailgunClient;
+    #errorHandler;
+
+    static BATCH_SIZE = 1000;
+
+    /**
+     * @param {object} dependencies
+     * @param {import('@tryghost/mailgun-client/lib/mailgun-client')} dependencies.mailgunClient - mailgun client to send emails
+     * @param {Function} [dependencies.errorHandler] - custom error handler for logging exceptions
+     */
+    constructor({
+        mailgunClient,
+        errorHandler
+    }) {
+        this.#mailgunClient = mailgunClient;
+        this.#errorHandler = errorHandler;
+    }
+
+    #createRecipientData(replacements) {
+        let recipientData = {};
+
+        recipientData = replacements.reduce((acc, replacement) => {
+            const {id, value} = replacement;
+            if (!acc[id]) {
+                acc[id] = {};
+            }
+            acc[id] = value;
+            return acc;
+        }, {});
+
+        return recipientData;
+    }
+
+    #updateRecipientVariables(data, replacementDefinitions) {
+        for (const def of replacementDefinitions) {
+            data = data.replace(
+                def.token,
+                `%recipient.${def.id}%`
+            );
+        }
+        return data;
+    }
+
+    /**
+     * Create mailgun error message for storing in the database
+     * @param {Object} error
+     * @param {string} error.message
+     * @param {string} error.details
+     * @returns {string}
+    */
+    #createMailgunErrorMessage(error) {
+        const message = (error?.message || '') + ':' + (error?.details || '');
+        return message.slice(0, 2000);
+    }
+
+    /**
+     * Send an email using the Mailgun API
+     * @param {import('./sending-service').EmailData} data
+     * @param {EmailSendingOptions} options
+     * @returns {Promise<EmailProviderSuccessResponse>}
+    */
+    async send(data, options) {
+        const {
+            subject,
+            html,
+            plaintext,
+            from,
+            replyTo,
+            emailId,
+            recipients,
+            replacementDefinitions
+        } = data;
+
+        logging.info(`Sending email to ${recipients.length} recipients`);
+        const startTime = Date.now();
+        debug(`sending message to ${recipients.length} recipients`);
+
+        try {
+            const messageData = {
+                subject,
+                html,
+                plaintext,
+                from,
+                replyTo,
+                id: emailId,
+                track_opens: !!options.openTrackingEnabled,
+                track_clicks: !!options.clickTrackingEnabled
+            };
+
+            // create recipient data for Mailgun using replacement definitions
+            const recipientData = recipients.reduce((acc, recipient) => {
+                acc[recipient.email] = this.#createRecipientData(recipient.replacements);
+                return acc;
+            }, {});
+
+            // update content to use Mailgun variable syntax for all replacements
+            ['html', 'plaintext'].forEach((key) => {
+                if (messageData[key]) {
+                    messageData[key] = this.#updateRecipientVariables(messageData[key], replacementDefinitions);
+                }
+            });
+
+            // send the email using Mailgun
+            // uses empty replacements array as we've already replaced all tokens with Mailgun variables
+            const response = await this.#mailgunClient.send(
+                messageData,
+                recipientData,
+                []
+            );
+
+            debug(`sent message (${Date.now() - startTime}ms)`);
+            logging.info(`Sent message (${Date.now() - startTime}ms)`);
+
+            // Return mailgun provider id, trim <> from response
+            return {
+                id: response.id.trim().replace(/^<|>$/g, '')
+            };
+        } catch ({error, messageData}) {
+            // REF: possible mailgun errors https://documentation.mailgun.com/en/latest/api-intro.html#status-codes
+            let ghostError = new errors.EmailError({
+                statusCode: error.status,
+                message: this.#createMailgunErrorMessage(error),
+                errorDetails: JSON.stringify({error, messageData}),
+                context: `Mailgun Error ${error.status}: ${error.details}`,
+                help: `https://ghost.org/docs/newsletters/#bulk-email-configuration`,
+                code: 'BULK_EMAIL_SEND_FAILED'
+            });
+
+            logging.warn(ghostError);
+            debug(`failed to send message (${Date.now() - startTime}ms)`);
+
+            // log error to custom error handler. ex sentry
+            this.#errorHandler(ghostError);
+            throw ghostError;
+        }
+    }
+}
+
+module.exports = MailgunEmailProvider;

ghost/email-service/lib/sending-service.js

@@ -4,8 +4,10 @@
  * @prop {string} plaintext
  * @prop {string} subject
  * @prop {string} from
+ * @prop {string} emailId
  * @prop {string} [replyTo]
  * @prop {Recipient[]} recipients
+ * @prop {import("./email-renderer").ReplacementDefinition[]} replacementDefinitions
  *
  * @typedef {object} IEmailProviderService
  * @prop {(emailData: EmailData, options: EmailSendingOptions) => Promise<EmailProviderSuccessResponse>} send
@@ -69,11 +71,12 @@ class SendingService {
      * @param {Post} data.post
      * @param {Newsletter} data.newsletter
      * @param {string|null} data.segment
+     * @param {string|null} data.emailId
      * @param {MemberLike[]} data.members
      * @param {EmailSendingOptions} options
      * @returns {Promise<EmailProviderSuccessResponse>}
     */
-    async send({post, newsletter, segment, members}, options) {
+    async send({post, newsletter, segment, members, emailId}, options) {
         const emailBody = await this.#emailRenderer.renderBody(
             post,
             newsletter,
@@ -88,7 +91,9 @@ class SendingService {
             replyTo: this.#emailRenderer.getReplyToAddress(post, newsletter) ?? undefined,
             html: emailBody.html,
             plaintext: emailBody.plaintext,
-            recipients
+            recipients,
+            emailId: emailId,
+            replacementDefinitions: emailBody.replacements
         }, options);
     }
 

ghost/email-service/test/mailgun-email-provider.test.js

@@ -0,0 +1,131 @@
+const MailgunEmailProvider = require('../lib/mailgun-email-provider');
+const sinon = require('sinon');
+const should = require('should');
+
+describe('Mailgun Email Provider', function () {
+    describe('send', function () {
+        let mailgunClient;
+        let sendStub;
+
+        beforeEach(function () {
+            sendStub = sinon.stub().resolves({
+                id: 'provider-123'
+            });
+
+            mailgunClient = {
+                send: sendStub
+            };
+        });
+
+        afterEach(function () {
+            sinon.restore();
+        });
+
+        it('calls mailgun client with correct data', async function () {
+            const mailgunEmailProvider = new MailgunEmailProvider({
+                mailgunClient,
+                errorHandler: () => {}
+            });
+
+            const response = await mailgunEmailProvider.send({
+                subject: 'Hi',
+                html: '<html><body>Hi {{name}}</body></html>',
+                plaintext: 'Hi',
+                from: 'ghost@example.com',
+                replyTo: 'ghost@example.com',
+                emailId: '123',
+                recipients: [
+                    {
+                        email: 'member@example.com',
+                        replacements: [
+                            {
+                                id: 'name',
+                                token: '{{name}}',
+                                value: 'John'
+                            }
+                        ]
+                    }
+                ],
+                replacementDefinitions: [
+                    {
+                        id: 'name',
+                        token: '{{name}}',
+                        getValue: () => 'John'
+                    }
+                ]
+            }, {
+                clickTrackingEnabled: true,
+                openTrackingEnabled: true
+            });
+            should(response.id).eql('provider-123');
+            should(sendStub.calledOnce).be.true();
+            sendStub.calledWith(
+                {
+                    subject: 'Hi',
+                    html: '<html><body>Hi %recipient.name%</body></html>',
+                    plaintext: 'Hi',
+                    from: 'ghost@example.com',
+                    replyTo: 'ghost@example.com',
+                    id: '123',
+                    track_opens: true,
+                    track_clicks: true
+                },
+                {'member@example.com': {name: 'John'}},
+                []
+            ).should.be.true();
+        });
+
+        it('handles mailgun client error correctly', async function () {
+            const mailgunErr = new Error('Bad Request');
+            mailgunErr.details = 'Invalid domain';
+            mailgunErr.status = 400;
+            sendStub = sinon.stub().throws({
+                error: mailgunErr,
+                messageData: {}
+            });
+
+            mailgunClient = {
+                send: sendStub
+            };
+
+            const mailgunEmailProvider = new MailgunEmailProvider({
+                mailgunClient,
+                errorHandler: () => {}
+            });
+            try {
+                const response = await mailgunEmailProvider.send({
+                    subject: 'Hi',
+                    html: '<html><body>Hi {{name}}</body></html>',
+                    plaintext: 'Hi',
+                    from: 'ghost@example.com',
+                    replyTo: 'ghost@example.com',
+                    emailId: '123',
+                    recipients: [
+                        {
+                            email: 'member@example.com',
+                            replacements: [
+                                {
+                                    id: 'name',
+                                    token: '{{name}}',
+                                    value: 'John'
+                                }
+                            ]
+                        }
+                    ],
+                    replacementDefinitions: [
+                        {
+                            id: 'name',
+                            token: '{{name}}',
+                            getValue: () => 'John'
+                        }
+                    ]
+                }, {});
+                should(response).be.undefined();
+            } catch (e) {
+                should(e.message).eql('Bad Request:Invalid domain');
+                should(e.statusCode).eql(400);
+                should(e.errorDetails).eql('{"error":{"details":"Invalid domain","status":400},"messageData":{}}');
+            }
+        });
+    });
+});