Ghost/ghost/admin/app/services/ajax.js

364 lines
12 KiB
JavaScript
Raw Normal View History

import AjaxService from 'ember-ajax/services/ajax';
import classic from 'ember-classic-decorator';
import config from 'ghost-admin/config/environment';
Made server unreachable and maintenance error request retries application-wide closes https://github.com/TryGhost/Team/issues/837 We previously added automatic retries to the editor controller for post saves; reviewing the resulting logs in Sentry we can see this stopped the "Server unreachable" error alerts showing to users because the requests typically succeeded on the first retry that was made 5 seconds later. However the problem is not limited to post saves and we can see other requests hitting the same issue, including when working in the editor such as adding embed cards, uploading images, or fetching member counts before publishing. All of the API network requests we make in Admin run through an `ajax` service that makes and handles the request/response. By moving the retry logic for specific errors out of the editor controller and into the ajax service we can make temporary connection handling more graceful across the app. - move retry behaviour from the editor controller to the `ajax` service so we can retry any request rather than just post save requests - speed up retries so we reconnect as soon as possible - first retry at 500ms, then every 1000ms (previous was every 5s which meant overly long waits) - reduce total retry time from >30s to 15s - improve reporting to Sentry - report when a retry was required - report when a retry failed - include the total time taken for both success and failure reports - include the `server` header value from requests to distinguish between CDNs - include type of error so we can distinguish "server unreachable" from "maintenance" retries
2021-06-30 16:51:40 +03:00
import moment from 'moment';
import {AjaxError, isAjaxError, isForbiddenError} from 'ember-ajax/errors';
Made server unreachable and maintenance error request retries application-wide closes https://github.com/TryGhost/Team/issues/837 We previously added automatic retries to the editor controller for post saves; reviewing the resulting logs in Sentry we can see this stopped the "Server unreachable" error alerts showing to users because the requests typically succeeded on the first retry that was made 5 seconds later. However the problem is not limited to post saves and we can see other requests hitting the same issue, including when working in the editor such as adding embed cards, uploading images, or fetching member counts before publishing. All of the API network requests we make in Admin run through an `ajax` service that makes and handles the request/response. By moving the retry logic for specific errors out of the editor controller and into the ajax service we can make temporary connection handling more graceful across the app. - move retry behaviour from the editor controller to the `ajax` service so we can retry any request rather than just post save requests - speed up retries so we reconnect as soon as possible - first retry at 500ms, then every 1000ms (previous was every 5s which meant overly long waits) - reduce total retry time from >30s to 15s - improve reporting to Sentry - report when a retry was required - report when a retry failed - include the total time taken for both success and failure reports - include the `server` header value from requests to distinguish between CDNs - include type of error so we can distinguish "server unreachable" from "maintenance" retries
2021-06-30 16:51:40 +03:00
import {captureMessage} from '@sentry/browser';
import {get} from '@ember/object';
import {isArray as isEmberArray} from '@ember/array';
import {isNone} from '@ember/utils';
import {inject as service} from '@ember/service';
Made server unreachable and maintenance error request retries application-wide closes https://github.com/TryGhost/Team/issues/837 We previously added automatic retries to the editor controller for post saves; reviewing the resulting logs in Sentry we can see this stopped the "Server unreachable" error alerts showing to users because the requests typically succeeded on the first retry that was made 5 seconds later. However the problem is not limited to post saves and we can see other requests hitting the same issue, including when working in the editor such as adding embed cards, uploading images, or fetching member counts before publishing. All of the API network requests we make in Admin run through an `ajax` service that makes and handles the request/response. By moving the retry logic for specific errors out of the editor controller and into the ajax service we can make temporary connection handling more graceful across the app. - move retry behaviour from the editor controller to the `ajax` service so we can retry any request rather than just post save requests - speed up retries so we reconnect as soon as possible - first retry at 500ms, then every 1000ms (previous was every 5s which meant overly long waits) - reduce total retry time from >30s to 15s - improve reporting to Sentry - report when a retry was required - report when a retry failed - include the total time taken for both success and failure reports - include the `server` header value from requests to distinguish between CDNs - include type of error so we can distinguish "server unreachable" from "maintenance" retries
2021-06-30 16:51:40 +03:00
import {timeout} from 'ember-concurrency';
2016-01-18 18:37:14 +03:00
const JSON_CONTENT_TYPE = 'application/json';
const GHOST_REQUEST = /\/ghost\/api\//;
function isJSONContentType(header) {
if (!header || isNone(header)) {
return false;
}
return header.indexOf(JSON_CONTENT_TYPE) === 0;
}
/* Version mismatch error */
export class VersionMismatchError extends AjaxError {
constructor(payload) {
super(payload, 'API server is running a newer version of Ghost, please upgrade.');
}
}
export function isVersionMismatchError(errorOrStatus, payload) {
if (isAjaxError(errorOrStatus)) {
return errorOrStatus instanceof VersionMismatchError;
} else {
return get(payload || {}, 'errors.firstObject.type') === 'VersionMismatchError';
}
}
/* Server unreachable error */
export class ServerUnreachableError extends AjaxError {
constructor(payload) {
super(payload, 'Server was unreachable');
}
}
export function isServerUnreachableError(error) {
if (isAjaxError(error)) {
return error instanceof ServerUnreachableError;
} else {
return error === 0 || error === '0';
}
}
/* Request entity too large error */
export class RequestEntityTooLargeError extends AjaxError {
constructor(payload) {
super(payload, 'Request is larger than the maximum file size the server allows');
}
}
export function isRequestEntityTooLargeError(errorOrStatus) {
if (isAjaxError(errorOrStatus)) {
return errorOrStatus instanceof RequestEntityTooLargeError;
} else {
return errorOrStatus === 413;
}
}
/* Unsupported media type error */
export class UnsupportedMediaTypeError extends AjaxError {
constructor(payload) {
super(payload, 'Request contains an unknown or unsupported file type.');
}
}
export function isUnsupportedMediaTypeError(errorOrStatus) {
if (isAjaxError(errorOrStatus)) {
return errorOrStatus instanceof UnsupportedMediaTypeError;
} else {
return errorOrStatus === 415;
}
}
/* Maintenance error */
export class MaintenanceError extends AjaxError {
constructor(payload) {
super(payload, 'Ghost is currently undergoing maintenance, please wait a moment then retry.');
}
}
export function isMaintenanceError(errorOrStatus) {
if (isAjaxError(errorOrStatus)) {
return errorOrStatus instanceof MaintenanceError;
} else {
return errorOrStatus === 503;
}
}
/* Theme validation error */
export class ThemeValidationError extends AjaxError {
constructor(payload) {
super(payload, 'Theme is not compatible or contains errors.');
}
}
export function isThemeValidationError(errorOrStatus, payload) {
if (isAjaxError(errorOrStatus)) {
return errorOrStatus instanceof ThemeValidationError;
} else {
return get(payload || {}, 'errors.firstObject.type') === 'ThemeValidationError';
}
}
/* Host limit reached/exceeded error */
export class HostLimitError extends AjaxError {
constructor(payload) {
super(payload, 'A hosting plan limit was reached or exceeded.');
}
}
export function isHostLimitError(errorOrStatus, payload) {
if (isAjaxError(errorOrStatus)) {
return errorOrStatus instanceof HostLimitError;
} else {
return get(payload || {}, 'errors.firstObject.type') === 'HostLimitError';
}
}
export class EmailError extends AjaxError {
constructor(payload) {
super(payload, 'Please verify your email settings');
}
}
export function isEmailError(errorOrStatus, payload) {
if (isAjaxError(errorOrStatus)) {
return errorOrStatus instanceof EmailError;
} else {
return get(payload || {}, 'errors.firstObject.type') === 'EmailError';
}
}
/* end: custom error types */
export class AcceptedResponse {
constructor(data) {
this.data = data;
}
}
export function isAcceptedResponse(errorOrStatus) {
if (errorOrStatus === 202) {
return true;
}
return false;
}
@classic
class ajaxService extends AjaxService {
@service config;
@service session;
2016-01-18 18:37:14 +03:00
// flag to tell our ESA authenticator not to try an invalidate DELETE request
// because it's been triggered by this service's 401 handling which means the
// DELETE would fail and get stuck in an infinite loop
// TODO: find a more elegant way to handle this
skipSessionDeletion = false;
get headers() {
return {
'X-Ghost-Version': config.APP.version,
'App-Pragma': 'no-cache'
};
}
2016-01-18 18:37:14 +03:00
init() {
super.init(...arguments);
if (this.isTesting === undefined) {
this.isTesting = config.environment === 'test';
}
}
Made server unreachable and maintenance error request retries application-wide closes https://github.com/TryGhost/Team/issues/837 We previously added automatic retries to the editor controller for post saves; reviewing the resulting logs in Sentry we can see this stopped the "Server unreachable" error alerts showing to users because the requests typically succeeded on the first retry that was made 5 seconds later. However the problem is not limited to post saves and we can see other requests hitting the same issue, including when working in the editor such as adding embed cards, uploading images, or fetching member counts before publishing. All of the API network requests we make in Admin run through an `ajax` service that makes and handles the request/response. By moving the retry logic for specific errors out of the editor controller and into the ajax service we can make temporary connection handling more graceful across the app. - move retry behaviour from the editor controller to the `ajax` service so we can retry any request rather than just post save requests - speed up retries so we reconnect as soon as possible - first retry at 500ms, then every 1000ms (previous was every 5s which meant overly long waits) - reduce total retry time from >30s to 15s - improve reporting to Sentry - report when a retry was required - report when a retry failed - include the total time taken for both success and failure reports - include the `server` header value from requests to distinguish between CDNs - include type of error so we can distinguish "server unreachable" from "maintenance" retries
2021-06-30 16:51:40 +03:00
async _makeRequest(hash) {
// ember-ajax recognizes `application/vnd.api+json` as a JSON-API request
Made server unreachable and maintenance error request retries application-wide closes https://github.com/TryGhost/Team/issues/837 We previously added automatic retries to the editor controller for post saves; reviewing the resulting logs in Sentry we can see this stopped the "Server unreachable" error alerts showing to users because the requests typically succeeded on the first retry that was made 5 seconds later. However the problem is not limited to post saves and we can see other requests hitting the same issue, including when working in the editor such as adding embed cards, uploading images, or fetching member counts before publishing. All of the API network requests we make in Admin run through an `ajax` service that makes and handles the request/response. By moving the retry logic for specific errors out of the editor controller and into the ajax service we can make temporary connection handling more graceful across the app. - move retry behaviour from the editor controller to the `ajax` service so we can retry any request rather than just post save requests - speed up retries so we reconnect as soon as possible - first retry at 500ms, then every 1000ms (previous was every 5s which meant overly long waits) - reduce total retry time from >30s to 15s - improve reporting to Sentry - report when a retry was required - report when a retry failed - include the total time taken for both success and failure reports - include the `server` header value from requests to distinguish between CDNs - include type of error so we can distinguish "server unreachable" from "maintenance" retries
2021-06-30 16:51:40 +03:00
// and formats appropriately, we want to handle `application/json` the same
if (isJSONContentType(hash.contentType) && hash.type !== 'GET') {
if (typeof hash.data === 'object') {
hash.data = JSON.stringify(hash.data);
}
}
hash.withCredentials = true;
Made server unreachable and maintenance error request retries application-wide closes https://github.com/TryGhost/Team/issues/837 We previously added automatic retries to the editor controller for post saves; reviewing the resulting logs in Sentry we can see this stopped the "Server unreachable" error alerts showing to users because the requests typically succeeded on the first retry that was made 5 seconds later. However the problem is not limited to post saves and we can see other requests hitting the same issue, including when working in the editor such as adding embed cards, uploading images, or fetching member counts before publishing. All of the API network requests we make in Admin run through an `ajax` service that makes and handles the request/response. By moving the retry logic for specific errors out of the editor controller and into the ajax service we can make temporary connection handling more graceful across the app. - move retry behaviour from the editor controller to the `ajax` service so we can retry any request rather than just post save requests - speed up retries so we reconnect as soon as possible - first retry at 500ms, then every 1000ms (previous was every 5s which meant overly long waits) - reduce total retry time from >30s to 15s - improve reporting to Sentry - report when a retry was required - report when a retry failed - include the total time taken for both success and failure reports - include the `server` header value from requests to distinguish between CDNs - include type of error so we can distinguish "server unreachable" from "maintenance" retries
2021-06-30 16:51:40 +03:00
// attempt retries for 15 seconds in two situations:
// 1. Server Unreachable error from the browser (code 0), typically from short internet blips
// 2. Maintenance error from Ghost, upgrade in progress so API is temporarily unavailable
let success = false;
let errorName = null;
let attempts = 0;
let startTime = new Date();
let retryingMs = 0;
const maxRetryingMs = 15_000;
const retryPeriods = [500, 1000];
const retryErrorChecks = [this.isServerUnreachableError, this.isMaintenanceError];
const getErrorData = () => {
const data = {
errorName,
attempts,
totalSeconds: moment().diff(moment(startTime), 'seconds')
};
if (this._responseServer) {
data.server = this._responseServer;
}
return data;
};
const makeRequest = super._makeRequest.bind(this);
Made server unreachable and maintenance error request retries application-wide closes https://github.com/TryGhost/Team/issues/837 We previously added automatic retries to the editor controller for post saves; reviewing the resulting logs in Sentry we can see this stopped the "Server unreachable" error alerts showing to users because the requests typically succeeded on the first retry that was made 5 seconds later. However the problem is not limited to post saves and we can see other requests hitting the same issue, including when working in the editor such as adding embed cards, uploading images, or fetching member counts before publishing. All of the API network requests we make in Admin run through an `ajax` service that makes and handles the request/response. By moving the retry logic for specific errors out of the editor controller and into the ajax service we can make temporary connection handling more graceful across the app. - move retry behaviour from the editor controller to the `ajax` service so we can retry any request rather than just post save requests - speed up retries so we reconnect as soon as possible - first retry at 500ms, then every 1000ms (previous was every 5s which meant overly long waits) - reduce total retry time from >30s to 15s - improve reporting to Sentry - report when a retry was required - report when a retry failed - include the total time taken for both success and failure reports - include the `server` header value from requests to distinguish between CDNs - include type of error so we can distinguish "server unreachable" from "maintenance" retries
2021-06-30 16:51:40 +03:00
while (retryingMs <= maxRetryingMs && !success) {
try {
const result = await makeRequest(hash);
success = true;
if (attempts !== 0 && this.config.get('sentry_dsn')) {
captureMessage('Request took multiple attempts', {extra: getErrorData()});
}
return result;
} catch (error) {
errorName = error.response?.constructor?.name;
retryingMs = (new Date()) - startTime;
// avoid retries in tests because it slows things down and is not expected in mocks
// isTesting can be overridden in individual tests if required
if (this.isTesting) {
throw error;
}
if (retryErrorChecks.some(check => check(error.response)) && retryingMs <= maxRetryingMs) {
await timeout(retryPeriods[attempts] || retryPeriods[retryPeriods.length - 1]);
attempts += 1;
} else if (attempts > 0 && this.config.get('sentry_dsn')) {
captureMessage('Request failed after multiple attempts', {extra: getErrorData()});
throw error;
} else {
throw error;
}
}
}
}
handleResponse(status, headers, payload, request) {
if (this.isVersionMismatchError(status, headers, payload)) {
return new VersionMismatchError(payload);
} else if (this.isServerUnreachableError(status, headers, payload)) {
return new ServerUnreachableError(payload);
} else if (this.isRequestEntityTooLargeError(status, headers, payload)) {
return new RequestEntityTooLargeError(payload);
} else if (this.isUnsupportedMediaTypeError(status, headers, payload)) {
return new UnsupportedMediaTypeError(payload);
} else if (this.isMaintenanceError(status, headers, payload)) {
return new MaintenanceError(payload);
} else if (this.isThemeValidationError(status, headers, payload)) {
return new ThemeValidationError(payload);
} else if (this.isHostLimitError(status, headers, payload)) {
return new HostLimitError(payload);
} else if (this.isEmailError(status, headers, payload)) {
return new EmailError(payload);
} else if (this.isAcceptedResponse(status)) {
return new AcceptedResponse(payload);
}
let isGhostRequest = GHOST_REQUEST.test(request.url);
let isAuthenticated = this.get('session.isAuthenticated');
let isUnauthorized = this.isUnauthorizedError(status, headers, payload);
let isForbidden = isForbiddenError(status, headers, payload);
Made server unreachable and maintenance error request retries application-wide closes https://github.com/TryGhost/Team/issues/837 We previously added automatic retries to the editor controller for post saves; reviewing the resulting logs in Sentry we can see this stopped the "Server unreachable" error alerts showing to users because the requests typically succeeded on the first retry that was made 5 seconds later. However the problem is not limited to post saves and we can see other requests hitting the same issue, including when working in the editor such as adding embed cards, uploading images, or fetching member counts before publishing. All of the API network requests we make in Admin run through an `ajax` service that makes and handles the request/response. By moving the retry logic for specific errors out of the editor controller and into the ajax service we can make temporary connection handling more graceful across the app. - move retry behaviour from the editor controller to the `ajax` service so we can retry any request rather than just post save requests - speed up retries so we reconnect as soon as possible - first retry at 500ms, then every 1000ms (previous was every 5s which meant overly long waits) - reduce total retry time from >30s to 15s - improve reporting to Sentry - report when a retry was required - report when a retry failed - include the total time taken for both success and failure reports - include the `server` header value from requests to distinguish between CDNs - include type of error so we can distinguish "server unreachable" from "maintenance" retries
2021-06-30 16:51:40 +03:00
// used when reporting connection errors, helps distinguish CDN
if (isGhostRequest) {
this._responseServer = headers.server;
}
if (isAuthenticated && isGhostRequest && (isUnauthorized || (isForbidden && payload.errors?.[0].message === 'Authorization failed'))) {
this.skipSessionDeletion = true;
this.session.invalidate();
}
return super.handleResponse(...arguments);
}
2016-01-18 18:37:14 +03:00
normalizeErrorResponse(status, headers, payload) {
if (payload && typeof payload === 'object') {
let errors = payload.error || payload.errors || payload.message || undefined;
if (errors) {
if (!isEmberArray(errors)) {
errors = [errors];
}
payload.errors = errors.map(function (error) {
if (typeof error === 'string') {
return {message: error};
} else {
return error;
}
});
}
2016-01-18 18:37:14 +03:00
}
return super.normalizeErrorResponse(status, headers, payload);
}
isVersionMismatchError(status, headers, payload) {
return isVersionMismatchError(status, payload);
}
isServerUnreachableError(status) {
return isServerUnreachableError(status);
}
isRequestEntityTooLargeError(status) {
return isRequestEntityTooLargeError(status);
}
isUnsupportedMediaTypeError(status) {
return isUnsupportedMediaTypeError(status);
}
isMaintenanceError(status, headers, payload) {
return isMaintenanceError(status, payload);
}
isThemeValidationError(status, headers, payload) {
return isThemeValidationError(status, payload);
}
isHostLimitError(status, headers, payload) {
return isHostLimitError(status, payload);
}
isEmailError(status, headers, payload) {
return isEmailError(status, payload);
}
isAcceptedResponse(status) {
return isAcceptedResponse(status);
2016-01-18 18:37:14 +03:00
}
}
// we need to reopen so that internal methods use the correct contentType
ajaxService.reopen({
contentType: 'application/json; charset=UTF-8'
});
export default ajaxService;