Ghost/ghost/email-analytics-service/lib/email-analytics-service.js

195 lines
6.9 KiB
JavaScript
Raw Normal View History

const EventProcessingResult = require('./event-processing-result');
const debug = require('@tryghost/debug')('services:email-analytics');
Added new email event processor (#15879) fixes https://github.com/TryGhost/Team/issues/2310 This moves the processing of the events from the event-processor to a new email-event-processor in the email-service package. - The `EmailEventProcessor` only translates events from providerId/emailId to their known emailId, memberId and recipientId, and dispatches the corresponding events. - Since `EmailEventProcessor` runs in a separate worker thread, we can't listen for the dispatched events on the main thread. To accomplish this communication, the events dispatched from the `EmailEventProcessor` class are 'posted' via the postMessage method and redispatched on the main thread. - A new `EmailEventStorage` class reacts to the email events and stores it in the database. This code mostly corresponds to the (now deleted) subclass of the old `EmailEventProcessor` - Updating a members last_seen_at timestamp has moved to the lastSeenAtUpdater. - Email events no longer store `ObjectID` because these are not encodable across threads via postMessage - Includes new E2E tests that test the storage of all supported Mailgun events. Note that in these tests we run the processing on the main thread instead of on a separate thread (couldn't do this because stubbing is not possible across threads) There are some missing pieces that will get added in later PRs (this PR focuses on porting the existing functionality): - Handling temporary failures/bounces - Capturing the error messages of bounce events
2022-11-29 13:15:19 +03:00
/**
* @typedef {import('@tryghost/email-service').EmailEventProcessor} EmailEventProcessor
*/
module.exports = class EmailAnalyticsService {
Added new email event processor (#15879) fixes https://github.com/TryGhost/Team/issues/2310 This moves the processing of the events from the event-processor to a new email-event-processor in the email-service package. - The `EmailEventProcessor` only translates events from providerId/emailId to their known emailId, memberId and recipientId, and dispatches the corresponding events. - Since `EmailEventProcessor` runs in a separate worker thread, we can't listen for the dispatched events on the main thread. To accomplish this communication, the events dispatched from the `EmailEventProcessor` class are 'posted' via the postMessage method and redispatched on the main thread. - A new `EmailEventStorage` class reacts to the email events and stores it in the database. This code mostly corresponds to the (now deleted) subclass of the old `EmailEventProcessor` - Updating a members last_seen_at timestamp has moved to the lastSeenAtUpdater. - Email events no longer store `ObjectID` because these are not encodable across threads via postMessage - Includes new E2E tests that test the storage of all supported Mailgun events. Note that in these tests we run the processing on the main thread instead of on a separate thread (couldn't do this because stubbing is not possible across threads) There are some missing pieces that will get added in later PRs (this PR focuses on porting the existing functionality): - Handling temporary failures/bounces - Capturing the error messages of bounce events
2022-11-29 13:15:19 +03:00
config;
settings;
queries;
eventProcessor;
providers;
/**
* @param {object} dependencies
Added new email event processor (#15879) fixes https://github.com/TryGhost/Team/issues/2310 This moves the processing of the events from the event-processor to a new email-event-processor in the email-service package. - The `EmailEventProcessor` only translates events from providerId/emailId to their known emailId, memberId and recipientId, and dispatches the corresponding events. - Since `EmailEventProcessor` runs in a separate worker thread, we can't listen for the dispatched events on the main thread. To accomplish this communication, the events dispatched from the `EmailEventProcessor` class are 'posted' via the postMessage method and redispatched on the main thread. - A new `EmailEventStorage` class reacts to the email events and stores it in the database. This code mostly corresponds to the (now deleted) subclass of the old `EmailEventProcessor` - Updating a members last_seen_at timestamp has moved to the lastSeenAtUpdater. - Email events no longer store `ObjectID` because these are not encodable across threads via postMessage - Includes new E2E tests that test the storage of all supported Mailgun events. Note that in these tests we run the processing on the main thread instead of on a separate thread (couldn't do this because stubbing is not possible across threads) There are some missing pieces that will get added in later PRs (this PR focuses on porting the existing functionality): - Handling temporary failures/bounces - Capturing the error messages of bounce events
2022-11-29 13:15:19 +03:00
* @param {EmailEventProcessor} dependencies.eventProcessor
*/
constructor({config, settings, queries, eventProcessor, providers}) {
this.config = config;
this.settings = settings;
this.queries = queries;
this.eventProcessor = eventProcessor;
this.providers = providers;
}
async fetchAll() {
const result = new EventProcessingResult();
const shouldFetchStats = await this.queries.shouldFetchStats();
if (!shouldFetchStats) {
debug('fetchAll: skipping - fetch requirements not met');
return result;
}
const startFetch = new Date();
debug('fetchAll: starting');
for (const [, provider] of Object.entries(this.providers)) {
const providerResults = await provider.fetchAll(this.processEventBatch.bind(this));
result.merge(providerResults);
}
debug(`fetchAll: finished (${Date.now() - startFetch}ms)`);
return result;
}
async fetchLatest({maxEvents = Infinity} = {}) {
const result = new EventProcessingResult();
const shouldFetchStats = await this.queries.shouldFetchStats();
if (!shouldFetchStats) {
debug('fetchLatest: skipping - fetch requirements not met');
return result;
}
const lastTimestamp = await this.queries.getLastSeenEventTimestamp();
const startFetch = new Date();
debug('fetchLatest: starting');
providersLoop:
for (const [, provider] of Object.entries(this.providers)) {
const providerResults = await provider.fetchLatest(lastTimestamp, this.processEventBatch.bind(this), {maxEvents});
result.merge(providerResults);
if (result.totalEvents >= maxEvents) {
break providersLoop;
}
}
debug(`fetchLatest: finished in ${Date.now() - startFetch}ms. Fetched ${result.totalEvents} events`);
return result;
}
async processEventBatch(events) {
const result = new EventProcessingResult();
for (const event of events) {
Added new email event processor (#15879) fixes https://github.com/TryGhost/Team/issues/2310 This moves the processing of the events from the event-processor to a new email-event-processor in the email-service package. - The `EmailEventProcessor` only translates events from providerId/emailId to their known emailId, memberId and recipientId, and dispatches the corresponding events. - Since `EmailEventProcessor` runs in a separate worker thread, we can't listen for the dispatched events on the main thread. To accomplish this communication, the events dispatched from the `EmailEventProcessor` class are 'posted' via the postMessage method and redispatched on the main thread. - A new `EmailEventStorage` class reacts to the email events and stores it in the database. This code mostly corresponds to the (now deleted) subclass of the old `EmailEventProcessor` - Updating a members last_seen_at timestamp has moved to the lastSeenAtUpdater. - Email events no longer store `ObjectID` because these are not encodable across threads via postMessage - Includes new E2E tests that test the storage of all supported Mailgun events. Note that in these tests we run the processing on the main thread instead of on a separate thread (couldn't do this because stubbing is not possible across threads) There are some missing pieces that will get added in later PRs (this PR focuses on porting the existing functionality): - Handling temporary failures/bounces - Capturing the error messages of bounce events
2022-11-29 13:15:19 +03:00
const batchResult = await this.processEvent(event);
result.merge(batchResult);
}
return result;
}
Added new email event processor (#15879) fixes https://github.com/TryGhost/Team/issues/2310 This moves the processing of the events from the event-processor to a new email-event-processor in the email-service package. - The `EmailEventProcessor` only translates events from providerId/emailId to their known emailId, memberId and recipientId, and dispatches the corresponding events. - Since `EmailEventProcessor` runs in a separate worker thread, we can't listen for the dispatched events on the main thread. To accomplish this communication, the events dispatched from the `EmailEventProcessor` class are 'posted' via the postMessage method and redispatched on the main thread. - A new `EmailEventStorage` class reacts to the email events and stores it in the database. This code mostly corresponds to the (now deleted) subclass of the old `EmailEventProcessor` - Updating a members last_seen_at timestamp has moved to the lastSeenAtUpdater. - Email events no longer store `ObjectID` because these are not encodable across threads via postMessage - Includes new E2E tests that test the storage of all supported Mailgun events. Note that in these tests we run the processing on the main thread instead of on a separate thread (couldn't do this because stubbing is not possible across threads) There are some missing pieces that will get added in later PRs (this PR focuses on porting the existing functionality): - Handling temporary failures/bounces - Capturing the error messages of bounce events
2022-11-29 13:15:19 +03:00
/**
*
* @param {{id: string, type: any; severity: any; recipientEmail: any; emailId: any; providerId: string; timestamp: Date; error: {code: number; message: string; enhandedCode: string|number} | null}} event
Added new email event processor (#15879) fixes https://github.com/TryGhost/Team/issues/2310 This moves the processing of the events from the event-processor to a new email-event-processor in the email-service package. - The `EmailEventProcessor` only translates events from providerId/emailId to their known emailId, memberId and recipientId, and dispatches the corresponding events. - Since `EmailEventProcessor` runs in a separate worker thread, we can't listen for the dispatched events on the main thread. To accomplish this communication, the events dispatched from the `EmailEventProcessor` class are 'posted' via the postMessage method and redispatched on the main thread. - A new `EmailEventStorage` class reacts to the email events and stores it in the database. This code mostly corresponds to the (now deleted) subclass of the old `EmailEventProcessor` - Updating a members last_seen_at timestamp has moved to the lastSeenAtUpdater. - Email events no longer store `ObjectID` because these are not encodable across threads via postMessage - Includes new E2E tests that test the storage of all supported Mailgun events. Note that in these tests we run the processing on the main thread instead of on a separate thread (couldn't do this because stubbing is not possible across threads) There are some missing pieces that will get added in later PRs (this PR focuses on porting the existing functionality): - Handling temporary failures/bounces - Capturing the error messages of bounce events
2022-11-29 13:15:19 +03:00
* @returns {Promise<EventProcessingResult>}
*/
async processEvent(event) {
if (event.type === 'delivered') {
const recipient = await this.eventProcessor.handleDelivered({emailId: event.emailId, providerId: event.providerId, email: event.recipientEmail}, event.timestamp);
if (recipient) {
return new EventProcessingResult({
delivered: 1,
emailIds: [recipient.emailId],
memberIds: [recipient.memberId]
});
}
return new EventProcessingResult({unprocessable: 1});
}
if (event.type === 'opened') {
const recipient = await this.eventProcessor.handleOpened({emailId: event.emailId, providerId: event.providerId, email: event.recipientEmail}, event.timestamp);
if (recipient) {
return new EventProcessingResult({
opened: 1,
emailIds: [recipient.emailId],
memberIds: [recipient.memberId]
});
}
return new EventProcessingResult({unprocessable: 1});
}
if (event.type === 'failed') {
if (event.severity === 'permanent') {
const recipient = await this.eventProcessor.handlePermanentFailed({emailId: event.emailId, providerId: event.providerId, email: event.recipientEmail}, {id: event.id, timestamp: event.timestamp, error: event.error});
Added new email event processor (#15879) fixes https://github.com/TryGhost/Team/issues/2310 This moves the processing of the events from the event-processor to a new email-event-processor in the email-service package. - The `EmailEventProcessor` only translates events from providerId/emailId to their known emailId, memberId and recipientId, and dispatches the corresponding events. - Since `EmailEventProcessor` runs in a separate worker thread, we can't listen for the dispatched events on the main thread. To accomplish this communication, the events dispatched from the `EmailEventProcessor` class are 'posted' via the postMessage method and redispatched on the main thread. - A new `EmailEventStorage` class reacts to the email events and stores it in the database. This code mostly corresponds to the (now deleted) subclass of the old `EmailEventProcessor` - Updating a members last_seen_at timestamp has moved to the lastSeenAtUpdater. - Email events no longer store `ObjectID` because these are not encodable across threads via postMessage - Includes new E2E tests that test the storage of all supported Mailgun events. Note that in these tests we run the processing on the main thread instead of on a separate thread (couldn't do this because stubbing is not possible across threads) There are some missing pieces that will get added in later PRs (this PR focuses on porting the existing functionality): - Handling temporary failures/bounces - Capturing the error messages of bounce events
2022-11-29 13:15:19 +03:00
if (recipient) {
return new EventProcessingResult({
permanentFailed: 1,
emailIds: [recipient.emailId],
memberIds: [recipient.memberId]
});
}
return new EventProcessingResult({unprocessable: 1});
} else {
const recipient = await this.eventProcessor.handleTemporaryFailed({emailId: event.emailId, providerId: event.providerId, email: event.recipientEmail}, {id: event.id, timestamp: event.timestamp, error: event.error});
Added new email event processor (#15879) fixes https://github.com/TryGhost/Team/issues/2310 This moves the processing of the events from the event-processor to a new email-event-processor in the email-service package. - The `EmailEventProcessor` only translates events from providerId/emailId to their known emailId, memberId and recipientId, and dispatches the corresponding events. - Since `EmailEventProcessor` runs in a separate worker thread, we can't listen for the dispatched events on the main thread. To accomplish this communication, the events dispatched from the `EmailEventProcessor` class are 'posted' via the postMessage method and redispatched on the main thread. - A new `EmailEventStorage` class reacts to the email events and stores it in the database. This code mostly corresponds to the (now deleted) subclass of the old `EmailEventProcessor` - Updating a members last_seen_at timestamp has moved to the lastSeenAtUpdater. - Email events no longer store `ObjectID` because these are not encodable across threads via postMessage - Includes new E2E tests that test the storage of all supported Mailgun events. Note that in these tests we run the processing on the main thread instead of on a separate thread (couldn't do this because stubbing is not possible across threads) There are some missing pieces that will get added in later PRs (this PR focuses on porting the existing functionality): - Handling temporary failures/bounces - Capturing the error messages of bounce events
2022-11-29 13:15:19 +03:00
if (recipient) {
return new EventProcessingResult({
temporaryFailed: 1,
emailIds: [recipient.emailId],
memberIds: [recipient.memberId]
});
}
return new EventProcessingResult({unprocessable: 1});
}
}
if (event.type === 'unsubscribed') {
const recipient = await this.eventProcessor.handleUnsubscribed({emailId: event.emailId, providerId: event.providerId, email: event.recipientEmail}, event.timestamp);
if (recipient) {
return new EventProcessingResult({
unsubscribed: 1,
emailIds: [recipient.emailId],
memberIds: [recipient.memberId]
});
}
return new EventProcessingResult({unprocessable: 1});
}
if (event.type === 'complained') {
const recipient = await this.eventProcessor.handleComplained({emailId: event.emailId, providerId: event.providerId, email: event.recipientEmail}, event.timestamp);
if (recipient) {
return new EventProcessingResult({
complained: 1,
emailIds: [recipient.emailId],
memberIds: [recipient.memberId]
});
}
return new EventProcessingResult({unprocessable: 1});
}
return new EventProcessingResult({unhandled: 1});
}
async aggregateStats({emailIds = [], memberIds = []}) {
for (const emailId of emailIds) {
await this.aggregateEmailStats(emailId);
}
for (const memberId of memberIds) {
await this.aggregateMemberStats(memberId);
}
}
async aggregateEmailStats(emailId) {
return this.queries.aggregateEmailStats(emailId);
}
async aggregateMemberStats(memberId) {
return this.queries.aggregateMemberStats(memberId);
}
};