Ghost/ghost/members-api/lib/repositories/event.js
Simon Backx fd91f7eebb
Added email sent events (#15682)
fixes https://github.com/TryGhost/Team/issues/2137

For the analytics page, we need the sent events to show up immediately
after sending an email. Otherwise we need to wait for emails to be
marked as received (which takes too long) before being able to show them
on the analytics page.

This adds the email_sent_event, which is hidden by default everywhere
and used on the analytics page.
2022-10-24 11:11:44 +02:00

764 lines
26 KiB
JavaScript

const errors = require('@tryghost/errors');
const nql = require('@tryghost/nql');
module.exports = class EventRepository {
constructor({
EmailRecipient,
MemberSubscribeEvent,
MemberPaymentEvent,
MemberStatusEvent,
MemberLoginEvent,
MemberCreatedEvent,
SubscriptionCreatedEvent,
MemberPaidSubscriptionEvent,
MemberLinkClickEvent,
MemberFeedback,
Comment,
labsService,
memberAttributionService
}) {
this._MemberSubscribeEvent = MemberSubscribeEvent;
this._MemberPaidSubscriptionEvent = MemberPaidSubscriptionEvent;
this._MemberPaymentEvent = MemberPaymentEvent;
this._MemberStatusEvent = MemberStatusEvent;
this._MemberLoginEvent = MemberLoginEvent;
this._EmailRecipient = EmailRecipient;
this._Comment = Comment;
this._labsService = labsService;
this._MemberCreatedEvent = MemberCreatedEvent;
this._SubscriptionCreatedEvent = SubscriptionCreatedEvent;
this._MemberLinkClickEvent = MemberLinkClickEvent;
this._MemberFeedback = MemberFeedback;
this._memberAttributionService = memberAttributionService;
}
async getEventTimeline(options = {}) {
if (!options.limit) {
options.limit = 10;
}
let filters = this.getNQLSubset(options.filter);
// Changing this order might need a change in the query functions
// because of the different underlying models.
options.order = 'created_at desc';
// Create a list of all events that can be queried
const pageActions = [
{type: 'comment_event', action: 'getCommentEvents'},
{type: 'click_event', action: 'getClickEvents'},
{type: 'signup_event', action: 'getSignupEvents'},
{type: 'subscription_event', action: 'getSubscriptionEvents'}
];
// Some events are not filterable by post_id
if (!filters['data.post_id']) {
pageActions.push(
{type: 'newsletter_event', action: 'getNewsletterSubscriptionEvents'},
{type: 'login_event', action: 'getLoginEvents'},
{type: 'payment_event', action: 'getPaymentEvents'}
);
}
if (this._EmailRecipient) {
pageActions.push({type: 'email_sent_event', action: 'getEmailSentEvents'});
pageActions.push({type: 'email_delivered_event', action: 'getEmailDeliveredEvents'});
pageActions.push({type: 'email_opened_event', action: 'getEmailOpenedEvents'});
pageActions.push({type: 'email_failed_event', action: 'getEmailFailedEvents'});
}
if (this._labsService.isSet('audienceFeedback')) {
pageActions.push({type: 'feedback_event', action: 'getFeedbackEvents'});
}
//Filter events to query
const filteredPages = filters.type ? pageActions.filter(page => nql(filters.type).queryJSON(page)) : pageActions;
//Start the promises
const pages = filteredPages.map(page => this[page.action](options, filters));
const allEventPages = await Promise.all(pages);
const allEvents = allEventPages.flatMap(page => page.data);
const totalEvents = allEventPages.reduce((accumulator, page) => accumulator + page.meta.pagination.total, 0);
return {
events: allEvents.sort(
(a, b) => {
return new Date(b.data.created_at).getTime() - new Date(a.data.created_at).getTime();
}
).slice(0, options.limit),
meta: {
pagination: {
limit: options.limit,
total: totalEvents,
pages: options.limit > 0 ? Math.ceil(totalEvents / options.limit) : null,
// Other values are unavailable (not possible to calculate easily)
page: null,
next: null,
prev: null
}
}
};
}
async registerPayment(data) {
await this._MemberPaymentEvent.add({
...data,
source: 'stripe'
});
}
async getNewsletterSubscriptionEvents(options = {}, filters = {}) {
options = {
...options,
withRelated: ['member', 'newsletter'],
filter: []
};
if (filters['data.created_at']) {
options.filter.push(filters['data.created_at'].replace(/data.created_at:/g, 'created_at:'));
}
if (filters['data.source']) {
options.filter.push(filters['data.source'].replace(/data.source:/g, 'source:'));
}
if (filters['data.member_id']) {
options.filter.push(filters['data.member_id'].replace(/data.member_id:/g, 'member_id:'));
}
options.filter = options.filter.join('+');
const {data: models, meta} = await this._MemberSubscribeEvent.findPage(options);
const data = models.map((model) => {
return {
type: 'newsletter_event',
data: model.toJSON(options)
};
});
return {
data,
meta
};
}
async getSubscriptionEvents(options = {}, filters = {}) {
if (!this._labsService.isSet('memberAttribution')){
options = {
...options,
withRelated: ['member'],
filter: []
};
if (filters['data.created_at']) {
options.filter.push(filters['data.created_at'].replace(/data.created_at:/g, 'created_at:'));
}
if (filters['data.member_id']) {
options.filter.push(filters['data.member_id'].replace(/data.member_id:/g, 'member_id:'));
}
options.filter = options.filter.join('+');
const {data: models, meta} = await this._MemberPaidSubscriptionEvent.findPage(options);
const data = models.map((model) => {
return {
type: 'subscription_event',
data: model.toJSON(options)
};
});
return {
data,
meta
};
}
options = {
...options,
withRelated: ['member', 'subscriptionCreatedEvent.postAttribution', 'subscriptionCreatedEvent.userAttribution', 'subscriptionCreatedEvent.tagAttribution'],
filter: []
};
if (filters['data.created_at']) {
options.filter.push(filters['data.created_at'].replace(/data.created_at:/g, 'created_at:'));
}
if (filters['data.member_id']) {
options.filter.push(filters['data.member_id'].replace(/data.member_id:/g, 'member_id:'));
}
if (filters['data.post_id']) {
options.filter.push(filters['data.post_id'].replace(/data.post_id:/g, 'subscriptionCreatedEvent.attribution_id:'));
options.filter.push('subscriptionCreatedEvent.attribution_type:post');
}
options.filter = options.filter.join('+');
const {data: models, meta} = await this._MemberPaidSubscriptionEvent.findPage(options);
const data = models.map((model) => {
return {
type: 'subscription_event',
data: {
...model.toJSON(options),
attribution: model.get('type') === 'created' && model.related('subscriptionCreatedEvent') && model.related('subscriptionCreatedEvent').id ? this._memberAttributionService.getEventAttribution(model.related('subscriptionCreatedEvent')) : null
}
};
});
return {
data,
meta
};
}
async getPaymentEvents(options = {}, filters = {}) {
options = {
...options,
withRelated: ['member'],
filter: []
};
if (filters['data.created_at']) {
options.filter.push(filters['data.created_at'].replace(/data.created_at:/g, 'created_at:'));
}
if (filters['data.member_id']) {
options.filter.push(filters['data.member_id'].replace(/data.member_id:/g, 'member_id:'));
}
options.filter = options.filter.join('+');
const {data: models, meta} = await this._MemberPaymentEvent.findPage(options);
const data = models.map((model) => {
return {
type: 'payment_event',
data: model.toJSON(options)
};
});
return {
data,
meta
};
}
async getLoginEvents(options = {}, filters = {}) {
options = {
...options,
withRelated: ['member'],
filter: []
};
if (filters['data.created_at']) {
options.filter.push(filters['data.created_at'].replace(/data.created_at:/g, 'created_at:'));
}
if (filters['data.member_id']) {
options.filter.push(filters['data.member_id'].replace(/data.member_id:/g, 'member_id:'));
}
options.filter = options.filter.join('+');
const {data: models, meta} = await this._MemberLoginEvent.findPage(options);
const data = models.map((model) => {
return {
type: 'login_event',
data: model.toJSON(options)
};
});
return {
data,
meta
};
}
async getSignupEvents(options = {}, filters = {}) {
if (!this._labsService.isSet('memberAttribution')){
options = {
...options,
withRelated: ['member'],
filter: ['from_status:null']
};
if (filters['data.created_at']) {
options.filter.push(filters['data.created_at'].replace(/data.created_at:/g, 'created_at:'));
}
if (filters['data.member_id']) {
options.filter.push(filters['data.member_id'].replace(/data.member_id:/g, 'member_id:'));
}
options.filter = options.filter.join('+');
const {data: models, meta} = await this._MemberStatusEvent.findPage(options);
const data = models.map((model) => {
return {
type: 'signup_event',
data: model.toJSON(options)
};
});
return {
data,
meta
};
}
return this.getCreatedEvents(options, filters);
}
async getCreatedEvents(options = {}, filters = {}) {
options = {
...options,
withRelated: ['member', 'postAttribution', 'userAttribution', 'tagAttribution'],
filter: []
};
if (filters['data.created_at']) {
options.filter.push(filters['data.created_at'].replace(/data.created_at:/g, 'created_at:'));
}
if (filters['data.member_id']) {
options.filter.push(filters['data.member_id'].replace(/data.member_id:/g, 'member_id:'));
}
if (filters['data.source']) {
options.filter.push(filters['data.source'].replace(/data.source:/g, 'source:'));
}
if (filters['data.post_id']) {
options.filter.push(filters['data.post_id'].replace(/data.post_id:/g, 'attribution_id:'));
options.filter.push('attribution_type:post');
}
options.filter = options.filter.join('+');
const {data: models, meta} = await this._MemberCreatedEvent.findPage(options);
const data = models.map((model) => {
return {
type: 'signup_event',
data: {
...model.toJSON(options),
attribution: this._memberAttributionService.getEventAttribution(model)
}
};
});
return {
data,
meta
};
}
async getCommentEvents(options = {}, filters = {}) {
options = {
...options,
withRelated: ['member', 'post', 'parent'],
filter: ['member_id:-null']
};
if (filters['data.created_at']) {
options.filter.push(filters['data.created_at'].replace(/data.created_at:/g, 'created_at:'));
}
if (filters['data.member_id']) {
options.filter.push(filters['data.member_id'].replace(/data.member_id:/g, 'member_id:'));
}
if (filters['data.post_id']) {
options.filter.push(filters['data.post_id'].replace(/data.post_id:/g, 'post_id:'));
}
options.filter = options.filter.join('+');
const {data: models, meta} = await this._Comment.findPage(options);
const data = models.map((model) => {
return {
type: 'comment_event',
data: model.toJSON(options)
};
});
return {
data,
meta
};
}
async getClickEvents(options = {}, filters = {}) {
options = {
...options,
withRelated: ['member', 'link', 'link.post'],
filter: []
};
if (filters['data.created_at']) {
options.filter.push(filters['data.created_at'].replace(/data.created_at:/g, 'created_at:'));
}
if (filters['data.member_id']) {
options.filter.push(filters['data.member_id'].replace(/data.member_id:/g, 'member_id:'));
}
if (filters['data.post_id']) {
options.filter.push(filters['data.post_id'].replace(/data.post_id:/g, 'post_id:'));
}
options.filter = options.filter.join('+');
const {data: models, meta} = await this._MemberLinkClickEvent.findPage(options);
const data = models.map((model) => {
return {
type: 'click_event',
data: model.toJSON(options)
};
});
return {
data,
meta
};
}
async getFeedbackEvents(options = {}, filters = {}) {
options = {
...options,
withRelated: ['member', 'post'],
filter: []
};
if (filters['data.created_at']) {
options.filter.push(filters['data.created_at'].replace(/data.created_at:/g, 'created_at:'));
}
if (filters['data.member_id']) {
options.filter.push(filters['data.member_id'].replace(/data.member_id:/g, 'member_id:'));
}
if (filters['data.post_id']) {
options.filter.push(filters['data.post_id'].replace(/data.post_id:/g, 'post_id:'));
}
options.filter = options.filter.join('+');
const {data: models, meta} = await this._MemberFeedback.findPage(options);
const data = models.map((model) => {
return {
type: 'feedback_event',
data: model.toJSON(options)
};
});
return {
data,
meta
};
}
async getEmailSentEvents(options = {}, filters = {}) {
options = {
...options,
withRelated: ['member', 'email'],
filter: ['failed_at:null', 'processed_at:-null']
};
if (filters['data.created_at']) {
options.filter.push(filters['data.created_at'].replace(/data.created_at:/g, 'processed_at:'));
}
if (filters['data.member_id']) {
options.filter.push(filters['data.member_id'].replace(/data.member_id:/g, 'member_id:'));
}
if (filters['data.post_id']) {
options.filter.push(filters['data.post_id'].replace(/data.post_id:/g, 'email.post_id:'));
}
options.filter = options.filter.join('+');
options.order = options.order.replace(/created_at/g, 'processed_at');
const {data: models, meta} = await this._EmailRecipient.findPage(
options
);
const data = models.map((model) => {
return {
type: 'email_sent_event',
data: {
member_id: model.get('member_id'),
created_at: model.get('processed_at'),
member: model.related('member').toJSON(),
email: model.related('email').toJSON()
}
};
});
return {
data,
meta
};
}
async getEmailDeliveredEvents(options = {}, filters = {}) {
options = {
...options,
withRelated: ['member', 'email'],
filter: ['delivered_at:-null']
};
if (filters['data.created_at']) {
options.filter.push(filters['data.created_at'].replace(/data.created_at:/g, 'delivered_at:'));
}
if (filters['data.member_id']) {
options.filter.push(filters['data.member_id'].replace(/data.member_id:/g, 'member_id:'));
}
if (filters['data.post_id']) {
options.filter.push(filters['data.post_id'].replace(/data.post_id:/g, 'email.post_id:'));
}
options.filter = options.filter.join('+');
options.order = options.order.replace(/created_at/g, 'delivered_at');
const {data: models, meta} = await this._EmailRecipient.findPage(
options
);
const data = models.map((model) => {
return {
type: 'email_delivered_event',
data: {
member_id: model.get('member_id'),
created_at: model.get('delivered_at'),
member: model.related('member').toJSON(),
email: model.related('email').toJSON()
}
};
});
return {
data,
meta
};
}
async getEmailOpenedEvents(options = {}, filters = {}) {
options = {
...options,
withRelated: ['member', 'email'],
filter: ['opened_at:-null']
};
if (filters['data.created_at']) {
options.filter.push(filters['data.created_at'].replace(/data.created_at:/g, 'opened_at:'));
}
if (filters['data.member_id']) {
options.filter.push(filters['data.member_id'].replace(/data.member_id:/g, 'member_id:'));
}
if (filters['data.post_id']) {
options.filter.push(filters['data.post_id'].replace(/data.post_id:/g, 'email.post_id:'));
}
options.filter = options.filter.join('+');
options.order = options.order.replace(/created_at/g, 'opened_at');
const {data: models, meta} = await this._EmailRecipient.findPage(
options
);
const data = models.map((model) => {
return {
type: 'email_opened_event',
data: {
member_id: model.get('member_id'),
created_at: model.get('opened_at'),
member: model.related('member').toJSON(),
email: model.related('email').toJSON()
}
};
});
return {
data,
meta
};
}
async getEmailFailedEvents(options = {}, filters = {}) {
options = {
...options,
withRelated: ['member', 'email'],
filter: ['failed_at:-null']
};
if (filters['data.created_at']) {
options.filter.push(filters['data.created_at'].replace(/data.created_at:/g, 'failed_at:'));
}
if (filters['data.member_id']) {
options.filter.push(filters['data.member_id'].replace(/data.member_id:/g, 'member_id:'));
}
if (filters['data.post_id']) {
options.filter.push(filters['data.post_id'].replace(/data.post_id:/g, 'email.post_id:'));
}
options.filter = options.filter.join('+');
options.order = options.order.replace(/created_at/g, 'failed_at');
const {data: models, meta} = await this._EmailRecipient.findPage(
options
);
const data = models.map((model) => {
return {
type: 'email_failed_event',
data: {
member_id: model.get('member_id'),
created_at: model.get('failed_at'),
member: model.related('member').toJSON(),
email: model.related('email').toJSON()
}
};
});
return {
data,
meta
};
}
/**
* Extract a subset of NQL.
* There are only a few properties allowed.
* Parenthesis are forbidden.
* Only ANDs are supported when combining properties.
*/
getNQLSubset(filter) {
if (!filter) {
return {};
}
const lex = nql(filter).lex();
const allowedFilters = ['type','data.created_at','data.member_id', 'data.post_id'];
const properties = lex
.filter(x => x.token === 'PROP')
.map(x => x.matched.slice(0, -1));
if (properties.some(prop => !allowedFilters.includes(prop))) {
throw new errors.IncorrectUsageError({
message: 'The only allowed filters are `type`, `data.created_at` and `data.member_id`'
});
}
if (lex.find(x => x.token === 'LPAREN')) {
throw new errors.IncorrectUsageError({
message: 'The filter can\'t contain parenthesis.'
});
}
const jsonFilter = nql(filter).toJSON();
const keys = Object.keys(jsonFilter);
if (keys.length === 1 && keys[0] === '$or') {
throw new errors.IncorrectUsageError({
message: 'The top level-filters can only combined with ANDs (+) and not ORs (,).'
});
}
// The filter is validated, it only contains one level of filters concatenated with `+`
const filters = filter.split('+');
/** @type {Object.<string, string>} */
let result = {};
for (const f of filters) {
// dirty way to parse a property, but it works according to https://github.com/NexesJS/NQL-Lang/blob/0e12d799a3a9c4d8651444e9284ce16c19cbc4f0/src/nql.l#L18
const key = f.split(':')[0];
if (!result[key]) {
result[key] = f;
} else {
result[key] += '+' + f;
}
}
return result;
}
async getSubscriptions() {
const results = await this._MemberSubscribeEvent.findAll({
aggregateSubscriptionDeltas: true
});
const resultsJSON = results.toJSON();
const cumulativeResults = resultsJSON.reduce((accumulator, result, index) => {
if (index === 0) {
return [{
date: result.date,
subscribed: result.subscribed_delta
}];
}
return accumulator.concat([{
date: result.date,
subscribed: result.subscribed_delta + accumulator[index - 1].subscribed
}]);
}, []);
return cumulativeResults;
}
async getMRR() {
const results = await this._MemberPaidSubscriptionEvent.findAll({
aggregateMRRDeltas: true
});
const resultsJSON = results.toJSON();
const cumulativeResults = resultsJSON.reduce((accumulator, result) => {
if (!accumulator[result.currency]) {
return {
...accumulator,
[result.currency]: [{
date: result.date,
mrr: result.mrr_delta,
currency: result.currency
}]
};
}
return {
...accumulator,
[result.currency]: accumulator[result.currency].concat([{
date: result.date,
mrr: result.mrr_delta + accumulator[result.currency].slice(-1)[0].mrr,
currency: result.currency
}])
};
}, {});
return cumulativeResults;
}
async getVolume() {
const results = await this._MemberPaymentEvent.findAll({
aggregatePaymentVolume: true
});
const resultsJSON = results.toJSON();
const cumulativeResults = resultsJSON.reduce((accumulator, result) => {
if (!accumulator[result.currency]) {
return {
...accumulator,
[result.currency]: [{
date: result.date,
volume: result.volume_delta,
currency: result.currency
}]
};
}
return {
...accumulator,
[result.currency]: accumulator[result.currency].concat([{
date: result.date,
volume: result.volume_delta + accumulator[result.currency].slice(-1)[0].volume,
currency: result.currency
}])
};
}, {});
return cumulativeResults;
}
async getStatuses() {
const results = await this._MemberStatusEvent.findAll({
aggregateStatusCounts: true
});
const resultsJSON = results.toJSON();
const cumulativeResults = resultsJSON.reduce((accumulator, result, index) => {
if (index === 0) {
return [{
date: result.date,
paid: result.paid_delta,
comped: result.comped_delta,
free: result.free_delta
}];
}
return accumulator.concat([{
date: result.date,
paid: result.paid_delta + accumulator[index - 1].paid,
comped: result.comped_delta + accumulator[index - 1].comped,
free: result.free_delta + accumulator[index - 1].free
}]);
}, []);
return cumulativeResults;
}
};