TryGhost/Ghost
1
0
Fork 0
mirror of https://github.com/TryGhost/Ghost.git synced 2025-01-07 11:30:55 +03:00
Code Issues Projects Releases Wiki Activity

Merge pull request #4673 from ErisDS/importer-imprv

Browse Source 
Data importer improvements and fixes
This commit is contained in:
Jason Williams 2014-12-18 14:11:41 -06:00
commit 8714822d48
15 changed files with 803 additions and 834 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
core/server/data/import/001.js
View File

@ -1,8 +0,0 @@
var Importer000 = require('./000');
module.exports = {
Importer001: Importer000,
importData: function (data) {
return new Importer000.importData(data);
}
};

8
core/server/data/import/002.js
View File

@ -1,8 +0,0 @@
var Importer000 = require('./000');
module.exports = {
Importer002: Importer000,
importData: function (data) {
return new Importer000.importData(data);
}
};

8
core/server/data/import/003.js
View File

@ -1,8 +0,0 @@
var Importer000 = require('./000');
module.exports = {
Importer003: Importer000,
importData: function (data) {
return new Importer000.importData(data);
}
};

44
core/server/data/import/000.js → core/server/data/import/data-importer.js
View File

@ -3,37 +3,15 @@ var Promise = require('bluebird'),
models = require('../../models'),
utils = require('./utils'),
Importer000;
DataImporter;
Importer000 = function () {
_.bindAll(this, 'doImport');
DataImporter = function () {};
this.version = '000';
this.importFrom = {
'000': this.doImport,
'001': this.doImport,
'002': this.doImport,
'003': this.doImport
};
DataImporter.prototype.importData = function (data) {
return this.doImport(data);
};
Importer000.prototype.importData = function (data) {
return this.canImport(data)
.then(function (importerFunc) {
return importerFunc(data);
});
};
Importer000.prototype.canImport = function (data) {
if (data.meta && data.meta.version && this.importFrom[data.meta.version]) {
return Promise.resolve(this.importFrom[data.meta.version]);
}
return Promise.reject('Unsupported version of data: ' + data.meta.version);
};
Importer000.prototype.loadUsers = function () {
DataImporter.prototype.loadUsers = function () {
var users = {all: {}};
return models.User.findAll({include: ['roles']}).then(function (_users) {
@ -52,11 +30,7 @@ Importer000.prototype.loadUsers = function () {
});
};
// Importer000.prototype.importerFunction = function (t) {
//
// };
Importer000.prototype.doUserImport = function (t, tableData, users, errors) {
DataImporter.prototype.doUserImport = function (t, tableData, users, errors) {
var userOps = [],
imported = [];
@ -89,7 +63,7 @@ Importer000.prototype.doUserImport = function (t, tableData, users, errors) {
return Promise.resolve({});
};
Importer000.prototype.doImport = function (data) {
DataImporter.prototype.doImport = function (data) {
var self = this,
tableData = data.data,
imported = {},
@ -171,8 +145,8 @@ Importer000.prototype.doImport = function (data) {
};
module.exports = {
Importer000: Importer000,
DataImporter: DataImporter,
importData: function (data) {
return new Importer000().importData(data);
return new DataImporter().importData(data);
}
};

25
core/server/data/import/index.js
View File

@ -3,7 +3,7 @@ var Promise = require('bluebird'),
validation = require('../validation'),
errors = require('../../errors'),
uuid = require('node-uuid'),
validator = require('validator'),
importer = require('./data-importer'),
tables = require('../schema').tables,
validate,
handleErrors,
@ -89,14 +89,14 @@ sanitize = function sanitize(data) {
});
_.each(tableNames, function (tableName) {
// Sanitize the table data for duplicates and valid uuid values
// Sanitize the table data for duplicates and valid uuid and created_at values
var sanitizedTableData = _.transform(data.data[tableName], function (memo, importValues) {
var uuidMissing = (!importValues.uuid && tables[tableName].uuid) ? true : false,
uuidMalformed = (importValues.uuid && !validator.isUUID(importValues.uuid)) ? true : false,
uuidMalformed = (importValues.uuid && !validation.validator.isUUID(importValues.uuid)) ? true : false,
isDuplicate,
problemTag;
// Check for correct UUID and fix if neccessary
// Check for correct UUID and fix if necessary
if (uuidMissing || uuidMalformed) {
importValues.uuid = uuid.v4();
}
@ -184,25 +184,12 @@ validate = function validate(data) {
});
};
module.exports = function (version, data) {
var importer,
sanitizeResults;
sanitizeResults = sanitize(data);
module.exports = function (data) {
var sanitizeResults = sanitize(data);
data = sanitizeResults.data;
return validate(data).then(function () {
try {
importer = require('./' + version);
} catch (ignore) {
// Zero effs given
}
if (!importer) {
return Promise.reject('No importer found');
}
return importer.importData(data);
}).then(function () {
return sanitizeResults;

27
core/server/data/import/utils.js
View File

@ -185,34 +185,25 @@ utils = {
var ops = [];
tableData = stripProperties(['id'], tableData);
_.each(tableData, function (post) {
// Validate minimum post fields
if (areEmpty(post, 'title', 'slug', 'markdown')) {
return;
}
ops.push(function () {
return models.Post.add(post, _.extend(internal, {transacting: transaction, importing: true}))
// The post importer has auto-timestamping disabled
if (!post.created_at) {
post.created_at = Date.now();
}
ops.push(models.Post.add(post, _.extend(internal, {transacting: transaction, importing: true}))
.catch(function (error) {
return Promise.reject({raw: error, model: 'post', data: post});
});
});
})
);
});
return Promise.reduce(ops, function (results, op) {
return op().then(function (result) {
results.push(result);
return results;
}).catch(function (error) {
if (error) {
results.push(error);
}
return results;
});
}, []).settle();
return Promise.settle(ops);
},
importUsers: function importUsers(tableData, existingUsers, transaction) {

6
core/server/data/importer/handlers/json.js
View File

@ -20,7 +20,11 @@ JSONHandler = {
importData = JSON.parse(fileData);
// if importData follows JSON-API format `{ db: [exportedData] }`
if (_.keys(importData).length === 1 && Array.isArray(importData.db)) {
if (_.keys(importData).length === 1) {
if (!importData.db || !Array.isArray(importData.db)) {
throw new Error('Invalid JSON format, expected `{ db: [exportedData] }`');
}
importData = importData.db[0];
}

2
core/server/data/importer/importers/data.js
View File

@ -8,7 +8,7 @@ DataImporter = {
return importData;
},
doImport: function (importData) {
return importer('003', importData);
return importer(importData);
}
};

6
core/server/data/validation/index.js
View File

@ -27,9 +27,8 @@ validator.extend('isEmptyOrURL', function (str) {
return (_.isEmpty(str) || validator.isURL(str, {require_protocol: false}));
});
// Validation validation against schema attributes
// values are checked against the validation objects
// form schema.js
// Validation against schema attributes
// values are checked against the validation objects from schema.js
validateSchema = function (tableName, model) {
var columns = _.keys(schema[tableName]),
validationErrors = [];
@ -163,6 +162,7 @@ validate = function (value, key, validations) {
};
module.exports = {
validator: validator,
validateSchema: validateSchema,
validateSettings: validateSettings,
validateActiveTheme: validateActiveTheme

1383
core/test/integration/import_spec.js
View File

File diff suppressed because it is too large Load Diff

35
core/test/unit/importer_spec.js
View File

@ -1,9 +1,10 @@
/*globals describe, afterEach, it*/
/*jshint expr:true*/
var should = require('should'),
sinon = require('sinon'),
Promise = require('bluebird'),
_ = require('lodash'),
var should = require('should'),
sinon = require('sinon'),
Promise = require('bluebird'),
_ = require('lodash'),
testUtils = require('../utils'),
// Stuff we are testing
ImportManager = require('../../server/data/importer'),
@ -179,6 +180,32 @@ describe('Importer', function () {
JSONHandler.types.should.containEql('application/json');
JSONHandler.loadFile.should.be.instanceof(Function);
});
it('correctly handles a valid db api wrapper', function (done) {
var file = [{
path: testUtils.fixtures.getExportFixturePath('export-003-api-wrapper'),
name: 'export-003-api-wrapper.json'
}];
JSONHandler.loadFile(file).then(function (result) {
_.keys(result).should.containEql('meta');
_.keys(result).should.containEql('data');
done();
});
});
it('correctly errors when given a bad db api wrapper', function (done) {
var file = [{
path: testUtils.fixtures.getExportFixturePath('export-003-api-wrapper-bad'),
name: 'export-003-api-wrapper-bad.json'
}];
JSONHandler.loadFile(file).then(function () {
done(new Error('Didn\'t error for bad db api wrapper'));
}).catch(function (response) {
response.type.should.equal('BadRequestError');
done();
});
});
});
describe('DataImporter', function () {

31
core/test/utils/fixtures/export-003-api-wrapper-bad.json Normal file
View File

@ -0,0 +1,31 @@
{"asdadas":[{
"meta": {
"exported_on": 1388318311015,
"version": "003"
},
"data": {
"posts": [
{
"id": 1,
"title": "Welcome to Ghost",
"slug": "welcome-to-ghost",
"markdown": "You're live! Nice.",
"html": "<p>You're live! Nice.</p>",
"image": null,
"featured": 0,
"page": 0,
"status": "published",
"language": "en_US",
"meta_title": null,
"meta_description": null,
"author_id": 2,
"created_at": 1388318310782,
"created_by": 1,
"updated_at": 1388318310782,
"updated_by": 1,
"published_at": 1388318310783,
"published_by": 2
}
]
}
}]}

31
core/test/utils/fixtures/export-003-api-wrapper.json Normal file
View File

@ -0,0 +1,31 @@
{"db":[{
"meta": {
"exported_on": 1388318311015,
"version": "003"
},
"data": {
"posts": [
{
"id": 1,
"title": "Welcome to Ghost",
"slug": "welcome-to-ghost",
"markdown": "You're live! Nice.",
"html": "<p>You're live! Nice.</p>",
"image": null,
"featured": 0,
"page": 0,
"status": "published",
"language": "en_US",
"meta_title": null,
"meta_description": null,
"author_id": 2,
"created_at": 1388318310782,
"created_by": 1,
"updated_at": 1388318310782,
"updated_by": 1,
"published_at": 1388318310783,
"published_by": 2
}
]
}
}]}

15
core/test/utils/fixtures/export-003-minimal.json Normal file
View File

@ -0,0 +1,15 @@
{
"meta": {
"exported_on": 1388318311015,
"version": "003"
},
"data": {
"posts": [
{
"title": "Welcome to Ghost",
"slug": "welcome-to-ghost",
"markdown": "You're live! Nice."
}
]
}
}

8
core/test/utils/index.js
View File

@ -280,11 +280,15 @@ fixtures = {
});
},
getExportFixturePath: function (filename) {
return path.resolve(__dirname + '/fixtures/' + filename + '.json');
},
loadExportFixture: function loadExportFixture(filename) {
var filepath = path.resolve(__dirname + '/fixtures/' + filename + '.json'),
var filePath = this.getExportFixturePath(filename),
readFile = Promise.promisify(fs.readFile);
return readFile(filepath).then(function (fileContents) {
return readFile(filePath).then(function (fileContents) {
var data;
// Parse the json data