Readded activity feed pagination tests (#15776)

fixes https://github.com/TryGhost/Team/issues/2218

The test should run a lot faster now.
This commit is contained in:
Simon Backx 2022-11-07 15:08:56 +01:00 committed by GitHub
parent b526c4b719
commit 231ceedc72
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 652 additions and 56 deletions

View File

@ -22951,6 +22951,606 @@ Object {
}
`;
exports[`Activity Feed API Filter-based pagination Can do filter based pagination for aggregated clicks for all posts 1: [body] 1`] = `
Object {
"events": Array [
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
],
"meta": Object {
"pagination": Object {
"limit": "32",
"next": null,
"page": null,
"pages": 2,
"prev": null,
"total": 33,
},
},
}
`;
exports[`Activity Feed API Filter-based pagination Can do filter based pagination for aggregated clicks for all posts 2: [headers] 1`] = `
Object {
"access-control-allow-origin": "http://127.0.0.1:2369",
"cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
"content-length": StringMatching /\\\\d\\+/,
"content-type": "application/json; charset=utf-8",
"etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
"vary": "Accept-Version, Origin, Accept-Encoding",
"x-powered-by": "Express",
}
`;
exports[`Activity Feed API Filter-based pagination Can do filter based pagination for aggregated clicks for all posts 3: [body] 1`] = `
Object {
"events": Array [
Object {
"data": Any<Object>,
"type": Any<String>,
},
],
"meta": Object {
"pagination": Object {
"limit": "32",
"next": null,
"page": null,
"pages": 1,
"prev": null,
"total": 1,
},
},
}
`;
exports[`Activity Feed API Filter-based pagination Can do filter based pagination for aggregated clicks for all posts 4: [headers] 1`] = `
Object {
"access-control-allow-origin": "http://127.0.0.1:2369",
"cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
"content-length": StringMatching /\\\\d\\+/,
"content-type": "application/json; charset=utf-8",
"etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
"vary": "Accept-Version, Origin, Accept-Encoding",
"x-powered-by": "Express",
}
`;
exports[`Activity Feed API Filter-based pagination Can do filter based pagination for aggregated clicks for one post 1: [body] 1`] = `
Object {
"events": Array [
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
],
"meta": Object {
"pagination": Object {
"limit": "8",
"next": null,
"page": null,
"pages": 2,
"prev": null,
"total": 9,
},
},
}
`;
exports[`Activity Feed API Filter-based pagination Can do filter based pagination for aggregated clicks for one post 2: [headers] 1`] = `
Object {
"access-control-allow-origin": "http://127.0.0.1:2369",
"cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
"content-length": StringMatching /\\\\d\\+/,
"content-type": "application/json; charset=utf-8",
"etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
"vary": "Accept-Version, Origin, Accept-Encoding",
"x-powered-by": "Express",
}
`;
exports[`Activity Feed API Filter-based pagination Can do filter based pagination for aggregated clicks for one post 3: [body] 1`] = `
Object {
"events": Array [
Object {
"data": Any<Object>,
"type": Any<String>,
},
],
"meta": Object {
"pagination": Object {
"limit": "8",
"next": null,
"page": null,
"pages": 1,
"prev": null,
"total": 1,
},
},
}
`;
exports[`Activity Feed API Filter-based pagination Can do filter based pagination for aggregated clicks for one post 4: [headers] 1`] = `
Object {
"access-control-allow-origin": "http://127.0.0.1:2369",
"cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
"content-length": StringMatching /\\\\d\\+/,
"content-type": "application/json; charset=utf-8",
"etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
"vary": "Accept-Version, Origin, Accept-Encoding",
"x-powered-by": "Express",
}
`;
exports[`Activity Feed API Filter-based pagination Can do filter based pagination for all posts 1: [body] 1`] = `
Object {
"events": Array [
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
],
"meta": Object {
"pagination": Object {
"limit": "36",
"next": null,
"page": null,
"pages": 2,
"prev": null,
"total": 37,
},
},
}
`;
exports[`Activity Feed API Filter-based pagination Can do filter based pagination for all posts 2: [headers] 1`] = `
Object {
"access-control-allow-origin": "http://127.0.0.1:2369",
"cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
"content-length": StringMatching /\\\\d\\+/,
"content-type": "application/json; charset=utf-8",
"etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
"vary": "Accept-Version, Origin, Accept-Encoding",
"x-powered-by": "Express",
}
`;
exports[`Activity Feed API Filter-based pagination Can do filter based pagination for all posts 3: [body] 1`] = `
Object {
"events": Array [
Object {
"data": Any<Object>,
"type": Any<String>,
},
],
"meta": Object {
"pagination": Object {
"limit": "36",
"next": null,
"page": null,
"pages": 1,
"prev": null,
"total": 1,
},
},
}
`;
exports[`Activity Feed API Filter-based pagination Can do filter based pagination for all posts 4: [headers] 1`] = `
Object {
"access-control-allow-origin": "http://127.0.0.1:2369",
"cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
"content-length": StringMatching /\\\\d\\+/,
"content-type": "application/json; charset=utf-8",
"etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
"vary": "Accept-Version, Origin, Accept-Encoding",
"x-powered-by": "Express",
}
`;
exports[`Activity Feed API Filter-based pagination Can do filter based pagination for one post 1: [body] 1`] = `
Object {
"events": Array [
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
],
"meta": Object {
"pagination": Object {
"limit": "10",
"next": null,
"page": null,
"pages": 2,
"prev": null,
"total": 13,
},
},
}
`;
exports[`Activity Feed API Filter-based pagination Can do filter based pagination for one post 2: [headers] 1`] = `
Object {
"access-control-allow-origin": "http://127.0.0.1:2369",
"cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
"content-length": StringMatching /\\\\d\\+/,
"content-type": "application/json; charset=utf-8",
"etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
"vary": "Accept-Version, Origin, Accept-Encoding",
"x-powered-by": "Express",
}
`;
exports[`Activity Feed API Filter-based pagination Can do filter based pagination for one post 3: [body] 1`] = `
Object {
"events": Array [
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
Object {
"data": Any<Object>,
"type": Any<String>,
},
],
"meta": Object {
"pagination": Object {
"limit": "10",
"next": null,
"page": null,
"pages": 1,
"prev": null,
"total": 3,
},
},
}
`;
exports[`Activity Feed API Filter-based pagination Can do filter based pagination for one post 4: [headers] 1`] = `
Object {
"access-control-allow-origin": "http://127.0.0.1:2369",
"cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
"content-length": StringMatching /\\\\d\\+/,
"content-type": "application/json; charset=utf-8",
"etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
"vary": "Accept-Version, Origin, Accept-Encoding",
"x-powered-by": "Express",
}
`;
exports[`Activity Feed API Returns click events in activity feed 1: [body] 1`] = `
Object {
"events": Array [

View File

@ -7,20 +7,54 @@ const moment = require('moment');
let agent;
async function testPagination(skippedTypes, postId, totalExpected) {
async function testPagination(skippedTypes, postId, totalExpected, limit) {
const postFilter = postId ? `+data.post_id:${postId}` : '';
// To make the test cover more edge cases, we test different limit configurations
for (let limit = 1; limit <= totalExpected; limit++) {
const {body: firstPage} = await agent
.get(`/members/events?filter=${encodeURIComponent(`type:-[${skippedTypes.join(',')}]${postFilter}`)}&limit=${limit}`)
const {body: firstPage} = await agent
.get(`/members/events?filter=${encodeURIComponent(`type:-[${skippedTypes.join(',')}]${postFilter}`)}&limit=${limit}`)
.expectStatus(200)
.matchHeaderSnapshot({
etag: anyEtag,
'content-length': anyContentLength // Depending on random conditions (ID generation) the order of events can change
})
.matchBodySnapshot({
events: new Array(limit).fill({
type: anyString,
data: anyObject
})
})
.expect(({body}) => {
if (postId) {
assert(!body.events.find(e => (e.data?.post?.id ?? e.data?.attribution?.id ?? e.data?.email?.post_id) !== postId && e.type !== 'aggregated_click_event'), 'Should only return events for the post');
}
// Assert total is correct
assert.equal(body.meta.pagination.total, totalExpected, 'Expected total of ' + totalExpected + ' at limit ' + limit);
});
let previousPage = firstPage;
let page = 1;
const allEvents = previousPage.events;
while (allEvents.length < totalExpected && page < 50) {
page += 1;
// Calculate next page
let lastId = previousPage.events[previousPage.events.length - 1].data.id;
let lastCreatedAt = moment(previousPage.events[previousPage.events.length - 1].data.created_at).format('YYYY-MM-DD HH:mm:ss');
const remaining = totalExpected - (page - 1) * limit;
const {body: secondPage} = await agent
.get(`/members/events?filter=${encodeURIComponent(`type:-[${skippedTypes.join(',')}]${postFilter}+(data.created_at:<'${lastCreatedAt}',(data.created_at:'${lastCreatedAt}'+id:<${lastId}))`)}&limit=${limit}`)
.expectStatus(200)
.matchHeaderSnapshot({
etag: anyEtag,
'content-length': anyContentLength // Depending on random conditions (ID generation) the order of events can change
})
.matchBodySnapshot({
events: new Array(limit).fill({
events: new Array(Math.min(remaining, limit)).fill({
type: anyString,
data: anyObject
})
@ -31,52 +65,16 @@ async function testPagination(skippedTypes, postId, totalExpected) {
}
// Assert total is correct
assert.equal(body.meta.pagination.total, totalExpected, 'Expected total of ' + totalExpected + ' at limit ' + limit);
assert.equal(body.meta.pagination.total, remaining, 'Expected total to be correct for page ' + page + ' with limit ' + limit);
});
let previousPage = firstPage;
let page = 1;
allEvents.push(...secondPage.events);
}
const allEvents = previousPage.events;
while (allEvents.length < totalExpected && page < 50) {
page += 1;
// Calculate next page
let lastId = previousPage.events[previousPage.events.length - 1].data.id;
let lastCreatedAt = moment(previousPage.events[previousPage.events.length - 1].data.created_at).format('YYYY-MM-DD HH:mm:ss');
const remaining = totalExpected - (page - 1) * limit;
const {body: secondPage} = await agent
.get(`/members/events?filter=${encodeURIComponent(`type:-[${skippedTypes.join(',')}]${postFilter}+(data.created_at:<'${lastCreatedAt}',(data.created_at:'${lastCreatedAt}'+id:<${lastId}))`)}&limit=${limit}`)
.expectStatus(200)
.matchHeaderSnapshot({
etag: anyEtag,
'content-length': anyContentLength // Depending on random conditions (ID generation) the order of events can change
})
.matchBodySnapshot({
events: new Array(Math.min(remaining, limit)).fill({
type: anyString,
data: anyObject
})
})
.expect(({body}) => {
if (postId) {
assert(!body.events.find(e => (e.data?.post?.id ?? e.data?.attribution?.id ?? e.data?.email?.post_id) !== postId && e.type !== 'aggregated_click_event'), 'Should only return events for the post');
}
// Assert total is correct
assert.equal(body.meta.pagination.total, remaining, 'Expected total to be correct for page ' + page + ' with limit ' + limit);
});
allEvents.push(...secondPage.events);
}
// Check if the ordering is correct and we didn't receive duplicate events
assert.equal(allEvents.length, totalExpected, 'Total actually received should match the total');
for (const event of allEvents) {
// Check no other events have the same id
assert.equal(allEvents.filter(e => e.data.id === event.data.id).length, 1);
}
// Check if the ordering is correct and we didn't receive duplicate events
assert.equal(allEvents.length, totalExpected, 'Total actually received should match the total');
for (const event of allEvents) {
// Check no other events have the same id
assert.equal(allEvents.filter(e => e.data.id === event.data.id).length, 1);
}
}
@ -187,15 +185,13 @@ describe('Activity Feed API', function () {
});
});
// Temporarily skip slow tests
// eslint-disable-next-line
describe.skip('Filter-based pagination', function () {
describe('Filter-based pagination', function () {
it('Can do filter based pagination for all posts', async function () {
// There is an annoying restriction in the pagination. It doesn't work for mutliple email events at the same time because they have the same id (causes issues as we use id to deduplicate the created_at timestamp)
// If that is ever fixed (it is difficult) we can update this test to not use a filter
// Same for click_event and aggregated_click_event (use same id)
const skippedTypes = ['email_opened_event', 'email_failed_event', 'email_delivered_event', 'aggregated_click_event'];
await testPagination(skippedTypes, null, 37);
await testPagination(skippedTypes, null, 37, 36);
});
it('Can do filter based pagination for one post', async function () {
@ -206,7 +202,7 @@ describe('Activity Feed API', function () {
// Same for click_event and aggregated_click_event (use same id)
const skippedTypes = ['email_opened_event', 'email_failed_event', 'email_delivered_event', 'aggregated_click_event'];
await testPagination(skippedTypes, postId, 13);
await testPagination(skippedTypes, postId, 13, 10);
});
it('Can do filter based pagination for aggregated clicks for one post', async function () {
@ -214,13 +210,13 @@ describe('Activity Feed API', function () {
const postId = fixtureManager.get('posts', 0).id;
const skippedTypes = ['email_opened_event', 'email_failed_event', 'email_sent_event', 'click_event'];
await testPagination(skippedTypes, postId, 9);
await testPagination(skippedTypes, postId, 9, 8);
});
it('Can do filter based pagination for aggregated clicks for all posts', async function () {
// Same as previous but with aggregated clicks instead of normal click events + email_delivered_events instead of sent events
const skippedTypes = ['email_opened_event', 'email_failed_event', 'email_sent_event', 'click_event'];
await testPagination(skippedTypes, null, 33);
await testPagination(skippedTypes, null, 33, 32);
});
});