Add validation for importer

closes #952
- moved api.js to api/index.js
- added api/db.js for import and export functions
- moved /ghost/debug/db/export to GET /api/v0.1/db
- moved /ghost/debug/db/import to POST /api/v0.1/db
- removed /ghost/debug/db/reset
- added validation for import
- added constraints object to migration

core/server.js

@@ -280,11 +280,7 @@ when(ghost.init()).then(function () {
     server.use(express.urlencoded());
     server.use('/ghost/upload/', express.multipart());
     server.use('/ghost/upload/', express.multipart({uploadDir: __dirname + '/content/images'}));
-    server.use('/ghost/debug/db/import/', express.multipart());
-
-    // Session handling
-    // Pro tip: while in development mode cookieSession can be used
-    // to keep you logged in while restarting the server
+    server.use('/api/v0.1/db/', express.multipart());
     server.use(express.cookieParser(ghost.dbHash));
     server.use(express.cookieSession({ cookie : { maxAge: 12 * 60 * 60 * 1000 }}));
 
@@ -335,7 +331,9 @@ when(ghost.init()).then(function () {
     // #### Notifications
     server.del('/api/v0.1/notifications/:id', authAPI, disableCachedResult, api.requestHandler(api.notifications.destroy));
     server.post('/api/v0.1/notifications/', authAPI, disableCachedResult, api.requestHandler(api.notifications.add));
-
+    // #### Import/Export
+    server.get('/api/v0.1/db/', auth, api.db['export']);
+    server.post('/api/v0.1/db/', auth, api.db['import']);
 
     // ### Admin routes
     /* TODO: put these somewhere in admin */
@@ -358,10 +356,10 @@ when(ghost.init()).then(function () {
     server.get('/ghost/content/', auth, admin.content);
     server.get('/ghost/settings*', auth, admin.settings);
     server.get('/ghost/debug/', auth, admin.debug.index);
-    server.get('/ghost/debug/db/export/', auth, admin.debug['export']);
-    server.post('/ghost/debug/db/import/', auth, admin.debug['import']);
+
     // We don't want to register bodyParser globally b/c of security concerns, so use multipart only here
     server.post('/ghost/upload/', auth, admin.uploader);
+
     // redirect to /ghost and let that do the authentication to prevent redirects to /ghost//admin etc.
     server.get(/^\/((ghost-admin|admin|wp-admin|dashboard|signin)\/?)/, function (req, res) {
         res.redirect('/ghost/');

core/server/api/db.js

@@ -0,0 +1,157 @@
+var Ghost         = require('../../ghost'),
+    dataExport    = require('../data/export'),
+    dataImport    = require('../data/import'),
+    api           = require('../api'),
+    fs            = require('fs-extra'),
+    path          = require('path'),
+    when          = require('when'),
+    nodefn        = require('when/node/function'),
+    _             = require('underscore'),
+
+    ghost         = new Ghost(),
+    db;
+
+db = {
+    export: function (req, res) {
+        return dataExport().then(function (exportedData) {
+            // Save the exported data to the file system for download
+            var fileName = path.resolve(__dirname + '/../../server/data/export/exported-' + (new Date().getTime()) + '.json');
+
+            return nodefn.call(fs.writeFile, fileName, JSON.stringify(exportedData)).then(function () {
+                return when(fileName);
+            });
+        }).then(function (exportedFilePath) {
+            // Send the exported data file
+            res.download(exportedFilePath, 'GhostData.json');
+        }).otherwise(function (error) {
+            // Notify of an error if it occurs
+            var notification = {
+                type: 'error',
+                message: error.message || error,
+                status: 'persistent',
+                id: 'per-' + (ghost.notifications.length + 1)
+            };
+
+            return api.notifications.add(notification).then(function () {
+                res.redirect('/ghost/debug/');
+            });
+        });
+    },
+    import: function (req, res) {
+
+        if (!req.files.importfile || req.files.importfile.size === 0 || req.files.importfile.name.indexOf('json') === -1) {
+            /**
+             * Notify of an error if it occurs
+             *
+             * - If there's no file (although if you don't select anything, the input is still submitted, so
+             *   !req.files.importfile will always be false)
+             * - If the size is 0
+             * - If the name doesn't have json in it
+             */
+            var notification = {
+                type: 'error',
+                message:  "Must select a .json file to import",
+                status: 'persistent',
+                id: 'per-' + (ghost.notifications.length + 1)
+            };
+
+            return api.notifications.add(notification).then(function () {
+                res.redirect('/ghost/debug/');
+            });
+        }
+
+        // Get the current version for importing
+        api.settings.read({ key: 'databaseVersion' })
+            .then(function (setting) {
+                return when(setting.value);
+            }, function () {
+                return when('001');
+            })
+            .then(function (databaseVersion) {
+                // Read the file contents
+                return nodefn.call(fs.readFile, req.files.importfile.path)
+                    .then(function (fileContents) {
+                        var importData,
+                            error = "",
+                            constraints = require('../data/migration/' + databaseVersion).constraints,
+                            constraintkeys = _.keys(constraints);
+
+                        // Parse the json data
+                        try {
+                            importData = JSON.parse(fileContents);
+                        } catch (e) {
+                            return when.reject(new Error("Failed to parse the import file"));
+                        }
+
+                        if (!importData.meta || !importData.meta.version) {
+                            return when.reject(new Error("Import data does not specify version"));
+                        }
+
+                        _.each(constraintkeys, function (constkey) {
+                            _.each(importData.data[constkey], function (elem) {
+                                var prop;
+                                for (prop in elem) {
+                                    if (elem.hasOwnProperty(prop)) {
+                                        if (constraints[constkey].hasOwnProperty(prop)) {
+                                            if (elem.hasOwnProperty(prop)) {
+                                                if (!_.isNull(elem[prop])) {
+                                                    if (elem[prop].length > constraints[constkey][prop].maxlength) {
+                                                        error += error !== "" ? "<br>" : "";
+                                                        error += "Property '" + prop + "' exceeds maximum length of " + constraints[constkey][prop].maxlength + " (element:" + constkey + " / id:" + elem.id + ")";
+                                                    }
+                                                } else {
+                                                    if (!constraints[constkey][prop].nullable) {
+                                                        error += error !== "" ? "<br>" : "";
+                                                        error += "Property '" + prop + "' is not nullable (element:" + constkey + " / id:" + elem.id + ")";
+                                                    }
+                                                }
+                                            }
+                                        } else {
+                                            error += error !== "" ? "<br>" : "";
+                                            error += "Property '" + prop + "' is not allowed (element:" + constkey + " / id:" + elem.id + ")";
+                                        }
+                                    }
+                                }
+                            });
+                        });
+
+                        if (error !== "") {
+                            return when.reject(new Error(error));
+                        }
+                        // Import for the current version
+                        return dataImport(databaseVersion, importData);
+                    });
+            })
+            .then(function importSuccess() {
+                var notification = {
+                    type: 'success',
+                    message: "Data imported. Log in with the user details you imported",
+                    status: 'persistent',
+                    id: 'per-' + (ghost.notifications.length + 1)
+                };
+
+                return api.notifications.add(notification).then(function () {
+                    delete req.session.user;
+                    res.set({
+                        "X-Cache-Invalidate": "/*"
+                    });
+                    res.redirect('/ghost/signin/');
+                });
+
+            }, function importFailure(error) {
+                // Notify of an error if it occurs
+                var notification = {
+                    type: 'error',
+                    message: error.message || error,
+                    status: 'persistent',
+                    id: 'per-' + (ghost.notifications.length + 1)
+                };
+
+                return api.notifications.add(notification).then(function () {
+                    res.redirect('/ghost/debug/');
+                });
+            });
+    },
+};
+
+module.exports.db = db;

core/server/api/index.js

@@ -1,11 +1,12 @@
 // # Ghost Data API
 // Provides access to the data model
 
-var Ghost        = require('../ghost'),
+var Ghost        = require('../../ghost'),
     _            = require('underscore'),
     when         = require('when'),
-    errors       = require('./errorHandling'),
-    permissions  = require('./permissions'),
+    errors       = require('../errorHandling'),
+    permissions  = require('../permissions'),
+    db           = require('./db'),
     canThis      = permissions.canThis,
 
     ghost        = new Ghost(),
@@ -405,4 +406,5 @@ module.exports.users = users;
 module.exports.tags = tags;
 module.exports.notifications = notifications;
 module.exports.settings = settings;
+module.exports.db = db.db;
 module.exports.requestHandler = requestHandler;

core/server/controllers/admin.js

@@ -1,11 +1,7 @@
 var Ghost         = require('../../ghost'),
-    dataExport    = require('../data/export'),
-    dataImport    = require('../data/import'),
     _             = require('underscore'),
     fs            = require('fs-extra'),
     path          = require('path'),
-    when          = require('when'),
-    nodefn        = require('when/node/function'),
     api           = require('../api'),
     moment        = require('moment'),
     errors        = require('../errorHandling'),
@@ -289,114 +285,6 @@ adminControllers = {
                 bodyClass: 'settings',
                 adminNav: setSelected(adminNavbar, 'settings')
             });
-        },
-        'export': function (req, res) {
-            return dataExport()
-                .then(function (exportedData) {
-                    // Save the exported data to the file system for download
-                    var fileName = path.resolve(__dirname + '/../../server/data/export/exported-' + (new Date().getTime()) + '.json');
-
-                    return nodefn.call(fs.writeFile, fileName, JSON.stringify(exportedData)).then(function () {
-                        return when(fileName);
-                    });
-                })
-                .then(function (exportedFilePath) {
-                    // Send the exported data file
-                    res.download(exportedFilePath, 'GhostData.json');
-                })
-                .otherwise(function (error) {
-                    // Notify of an error if it occurs
-                    var notification = {
-                        type: 'error',
-                        message: error.message || error,
-                        status: 'persistent',
-                        id: 'per-' + (ghost.notifications.length + 1)
-                    };
-
-                    return api.notifications.add(notification).then(function () {
-                        res.redirect('/ghost/debug/');
-                    });
-                });
-        },
-        'import': function (req, res) {
-            if (!req.files.importfile || req.files.importfile.size === 0 || req.files.importfile.name.indexOf('json') === -1) {
-                /**
-                 * Notify of an error if it occurs
-                 *
-                 * - If there's no file (although if you don't select anything, the input is still submitted, so
-                 *   !req.files.importfile will always be false)
-                 * - If the size is 0
-                 * - If the name doesn't have json in it
-                 */
-                var notification = {
-                    type: 'error',
-                    message:  "Must select a .json file to import",
-                    status: 'persistent',
-                    id: 'per-' + (ghost.notifications.length + 1)
-                };
-
-                return api.notifications.add(notification).then(function () {
-                    res.redirect('/ghost/debug/');
-                });
-            }
-
-            // Get the current version for importing
-            api.settings.read({ key: 'databaseVersion' })
-                .then(function (setting) {
-                    return when(setting.value);
-                }, function () {
-                    return when('001');
-                })
-                .then(function (databaseVersion) {
-                    // Read the file contents
-                    return nodefn.call(fs.readFile, req.files.importfile.path)
-                        .then(function (fileContents) {
-                            var importData;
-
-                            // Parse the json data
-                            try {
-                                importData = JSON.parse(fileContents);
-                            } catch (e) {
-                                return when.reject(new Error("Failed to parse the import file"));
-                            }
-
-                            if (!importData.meta || !importData.meta.version) {
-                                return when.reject(new Error("Import data does not specify version"));
-                            }
-
-                            // Import for the current version
-                            return dataImport(databaseVersion, importData);
-                        });
-                })
-                .then(function importSuccess() {
-                    var notification = {
-                        type: 'success',
-                        message: "Data imported. Log in with the user details you imported",
-                        status: 'persistent',
-                        id: 'per-' + (ghost.notifications.length + 1)
-                    };
-
-                    return api.notifications.add(notification).then(function () {
-                        req.session = null;
-                        res.set({
-                            "X-Cache-Invalidate": "/*"
-                        });
-                        res.redirect('/ghost/signin/');
-                    });
-
-                }, function importFailure(error) {
-                    // Notify of an error if it occurs
-                    var notification = {
-                        type: 'error',
-                        message: error.message || error,
-                        status: 'persistent',
-                        id: 'per-' + (ghost.notifications.length + 1)
-                    };
-
-                    return api.notifications.add(notification).then(function () {
-                        res.redirect('/ghost/debug/');
-                    });
-                });
         }
     }
 };

core/server/data/migration/000.js

@@ -1,7 +1,122 @@
 var when    = require('when'),
     knex    = require('../../models/base').knex,
     up,
-    down;
+    down,
+    constraints = {
+        posts: {
+            id: {maxlength: 0, nullable: false},
+            uuid: {maxlength: 36, nullable: false},
+            title: {maxlength: 150, nullable: false},
+            slug: {maxlength: 150, nullable: false},
+            markdown: {maxlength: 16777215, nullable: true},
+            html: {maxlength: 16777215, nullable: true},
+            image: {maxlength: 2000, nullable: true},
+            featured: {maxlength: 0, nullable: false},
+            page: {maxlength: 0, nullable: false},
+            status: {maxlength: 150, nullable: false},
+            language: {maxlength: 6, nullable: false},
+            meta_title: {maxlength: 150, nullable: true},
+            meta_description: {maxlength: 200, nullable: true},
+            author_id: {maxlength: 0, nullable: false},
+            created_at: {maxlength: 0, nullable: false},
+            created_by: {maxlength: 0, nullable: false},
+            updated_at: {maxlength: 0, nullable: true},
+            updated_by: {maxlength: 0, nullable: true},
+            published_at: {maxlength: 0, nullable: true},
+            published_by: {maxlength: 0, nullable: true},
+        },
+        users: {
+            id: {maxlength: 0, nullable: false},
+            uuid: {maxlength: 36, nullable: false},
+            name: {maxlength: 150, nullable: false},
+            slug: {maxlength: 150, nullable: false},
+            password: {maxlength: 60, nullable: false},
+            email: {maxlength: 254, nullable: false},
+            image: {maxlength: 2000, nullable: true},
+            cover: {maxlength: 2000, nullable: true},
+            bio: {maxlength: 200, nullable: true},
+            website: {maxlength: 2000, nullable: true},
+            location: {maxlength: 65535, nullable: true},
+            accessibility: {maxlength: 65535, nullable: true},
+            status: {maxlength: 150, nullable: false},
+            language: {maxlength: 6, nullable: false},
+            meta_title: {maxlength: 150, nullable: true},
+            meta_description: {maxlength: 200, nullable: true},
+            last_login: {maxlength: 0, nullable: true},
+            created_at: {maxlength: 0, nullable: false},
+            created_by: {maxlength: 0, nullable: false},
+            updated_at: {maxlength: 0, nullable: true},
+            updated_by: {maxlength: 0, nullable: true},
+        },
+        roles: {
+            id: {maxlength: 0, nullable: false},
+            uuid: {maxlength: 36, nullable: false},
+            name: {maxlength: 150, nullable: false},
+            description: {maxlength: 200, nullable: true},
+            created_at: {maxlength: 0, nullable: false},
+            created_by: {maxlength: 0, nullable: false},
+            updated_at: {maxlength: 0, nullable: true},
+            updated_by: {maxlength: 0, nullable: true},
+        },
+        roles_users: {
+            id: {maxlength: 0, nullable: false},
+            role_id: {maxlength: 0, nullable: false},
+            user_id: {maxlength: 0, nullable: false},
+        },
+        permissions: {
+            id: {maxlength: 0, nullable: false},
+            uuid: {maxlength: 36, nullable: false},
+            name: {maxlength: 150, nullable: false},
+            object_type: {maxlength: 150, nullable: false},
+            action_type: {maxlength: 150, nullable: false},
+            object_id: {maxlength: 0, nullable: true},
+            created_at: {maxlength: 0, nullable: false},
+            created_by: {maxlength: 0, nullable: false},
+            updated_at: {maxlength: 0, nullable: true},
+            updated_by: {maxlength: 0, nullable: true},
+        },
+        permissions_users: {
+            id: {maxlength: 0, nullable: false},
+            user_id: {maxlength: 0, nullable: false},
+            permission_id: {maxlength: 0, nullable: false},
+        },
+        permissions_roles: {
+            id: {maxlength: 0, nullable: false},
+            role_id: {maxlength: 0, nullable: false},
+            permission_id: {maxlength: 0, nullable: false},
+        },
+        settings: {
+            id: {maxlength: 0, nullable: false},
+            uuid: {maxlength: 36, nullable: false},
+            key: {maxlength: 150, nullable: false},
+            value: {maxlength: 65535, nullable: true},
+            type: {maxlength: 150, nullable: false},
+            created_at: {maxlength: 0, nullable: false},
+            created_by: {maxlength: 0, nullable: false},
+            updated_at: {maxlength: 0, nullable: true},
+            updated_by: {maxlength: 0, nullable: true},
+        },
+        tags: {
+            id: {maxlength: 0, nullable: false},
+            uuid: {maxlength: 36, nullable: false},
+            name: {maxlength: 150, nullable: false},
+            slug: {maxlength: 150, nullable: false},
+            description: {maxlength: 200, nullable: true},
+            parent_id: {maxlength: 0, nullable: true},
+            meta_title: {maxlength: 150, nullable: true},
+            meta_description: {maxlength: 200, nullable: true},
+            created_at: {maxlength: 0, nullable: false},
+            created_by: {maxlength: 0, nullable: false},
+            updated_at: {maxlength: 0, nullable: true},
+            updated_by: {maxlength: 0, nullable: true},
+        },
+        posts_tags: {
+            id: {maxlength: 0, nullable: false},
+            post_id: {maxlength: 0, nullable: false},
+            tag_id: {maxlength: 0, nullable: false},
+        }
+    };
+
 
 up = function () {
 
@@ -9,18 +124,18 @@ up = function () {
 
         knex.schema.createTable('posts', function (t) {
             t.increments().primary();
-            t.string('uuid', 36).notNull();
-            t.string('title', 150).notNull();
-            t.string('slug', 150).notNull().unique();
+            t.string('uuid', constraints.posts.uuid.maxlength).notNull();
+            t.string('title', constraints.posts.title.maxlength).notNull();
+            t.string('slug', constraints.posts.slug.maxlength).notNull().unique();
             t.text('markdown', 'medium').nullable();                    // max-length 16777215
             t.text('html', 'medium').nullable();                        // max-length 16777215
             t.text('image').nullable();                                 // max-length 2000
             t.bool('featured').notNull().defaultTo(false);
             t.bool('page').notNull().defaultTo(false);
-            t.string('status', 150).notNull().defaultTo('draft');
-            t.string('language', 6).notNull().defaultTo('en_US');
-            t.string('meta_title', 150).nullable();
-            t.string('meta_description', 200).nullable();
+            t.string('status', constraints.posts.status.maxlength).notNull().defaultTo('draft');
+            t.string('language', constraints.posts.language.maxlength).notNull().defaultTo('en_US');
+            t.string('meta_title', constraints.posts.meta_title.maxlength).nullable();
+            t.string('meta_description', constraints.posts.meta_description.maxlength).nullable();
             t.integer('author_id').notNull();
             t.dateTime('created_at').notNull();
             t.integer('created_by').notNull();
@@ -32,21 +147,21 @@ up = function () {
 
         knex.schema.createTable('users', function (t) {
             t.increments().primary();
-            t.string('uuid', 36).notNull();
-            t.string('name', 150).notNull();
-            t.string('slug', 150).notNull().unique();
-            t.string('password', 60).notNull();
-            t.string('email', 254).notNull().unique();
+            t.string('uuid', constraints.users.uuid.maxlength).notNull();
+            t.string('name', constraints.users.name.maxlength).notNull();
+            t.string('slug', constraints.users.slug.maxlength).notNull().unique();
+            t.string('password', constraints.users.password.maxlength).notNull();
+            t.string('email', constraints.users.email.maxlength).notNull().unique();
             t.text('image').nullable();                                 // max-length 2000
             t.text('cover').nullable();                                 // max-length 2000
-            t.string('bio', 200).nullable();
+            t.string('bio', constraints.users.bio.maxlength).nullable();
             t.text('website').nullable();                               // max-length 2000
             t.text('location').nullable();                              // max-length 65535
             t.text('accessibility').nullable();                         // max-length 65535
-            t.string('status', 150).notNull().defaultTo('active');
-            t.string('language', 6).notNull().defaultTo('en_US');
-            t.string('meta_title', 150).nullable();
-            t.string('meta_description', 200).nullable();
+            t.string('status', constraints.users.status.maxlength).notNull().defaultTo('active');
+            t.string('language', constraints.users.language.maxlength).notNull().defaultTo('en_US');
+            t.string('meta_title', constraints.users.meta_title.maxlength).nullable();
+            t.string('meta_description', constraints.users.meta_description.maxlength).nullable();
             t.dateTime('last_login').nullable();
             t.dateTime('created_at').notNull();
             t.integer('created_by').notNull();
@@ -56,9 +171,9 @@ up = function () {
 
         knex.schema.createTable('roles', function (t) {
             t.increments().primary();
-            t.string('uuid', 36).notNull();
-            t.string('name', 150).notNull();
-            t.string('description', 200).nullable();
+            t.string('uuid', constraints.roles.uuid.maxlength).notNull();
+            t.string('name', constraints.roles.name.maxlength).notNull();
+            t.string('description', constraints.roles.description.maxlength).nullable();
             t.dateTime('created_at').notNull();
             t.integer('created_by').notNull();
             t.dateTime('updated_at').nullable();
@@ -73,10 +188,10 @@ up = function () {
 
         knex.schema.createTable('permissions', function (t) {
             t.increments().primary();
-            t.string('uuid', 36).notNull();
-            t.string('name', 150).notNull();
-            t.string('object_type', 150).notNull();
-            t.string('action_type', 150).notNull();
+            t.string('uuid', constraints.permissions.uuid.maxlength).notNull();
+            t.string('name', constraints.permissions.name.maxlength).notNull();
+            t.string('object_type', constraints.permissions.object_type.maxlength).notNull();
+            t.string('action_type', constraints.permissions.action_type.maxlength).notNull();
             t.integer('object_id').nullable();
             t.dateTime('created_at').notNull();
             t.integer('created_by').notNull();
@@ -98,10 +213,10 @@ up = function () {
 
         knex.schema.createTable('settings', function (t) {
             t.increments().primary();
-            t.string('uuid', 36).notNull();
-            t.string('key', 150).notNull().unique();
+            t.string('uuid', constraints.settings.uuid.maxlength).notNull();
+            t.string('key', constraints.settings.key.maxlength).notNull().unique();
             t.text('value').nullable();                             // max-length 65535
-            t.string('type', 150).notNull().defaultTo('core');
+            t.string('type', constraints.settings.type.maxlength).notNull().defaultTo('core');
             t.dateTime('created_at').notNull();
             t.integer('created_by').notNull();
             t.dateTime('updated_at').nullable();
@@ -109,13 +224,13 @@ up = function () {
         }),
         knex.schema.createTable('tags', function (t) {
             t.increments().primary();
-            t.string('uuid', 36).notNull();
-            t.string('name', 150).notNull();
-            t.string('slug', 150).notNull().unique();
-            t.string('description', 200).nullable();
+            t.string('uuid', constraints.tags.uuid.maxlength).notNull();
+            t.string('name', constraints.tags.name.maxlength).notNull();
+            t.string('slug', constraints.tags.slug.maxlength).notNull().unique();
+            t.string('description', constraints.tags.description.maxlength).nullable();
             t.integer('parent_id').nullable();
-            t.string('meta_title', 150).nullable();
-            t.string('meta_description', 200).nullable();
+            t.string('meta_title', constraints.tags.meta_title.maxlength).nullable();
+            t.string('meta_description', constraints.tags.meta_description.maxlength).nullable();
             t.dateTime('created_at').notNull();
             t.integer('created_by').notNull();
             t.dateTime('updated_at').nullable();
@@ -150,4 +265,5 @@ down = function () {
 };
 
 exports.up   = up;
-exports.down = down;
+exports.down = down;
+exports.constraints = constraints;

core/server/data/migration/index.js

@@ -1,4 +1,3 @@
-
 var _               = require('underscore'),
     when            = require('when'),
     series          = require('when/sequence'),

core/server/views/debug.hbs

@@ -20,12 +20,12 @@
                 <fieldset>
                     <div class="form-group">
                         <label>Export</label>
-                        <a href="/ghost/debug/db/export/" class="button-save">Export</a>
+                        <a href="/api/v0.1/db/" class="button-save">Export</a>
                         <p>Export the blog settings and data.</p>
                     </div>
                 </fieldset>
             </form>
-            <form id="settings-import" method="post" action="/ghost/debug/db/import/" enctype="multipart/form-data">
+            <form id="settings-import" method="post" action="/api/v0.1/db/" enctype="multipart/form-data">
                 <input type="hidden" name="_csrf" value="{{csrfToken}}" />
                 <fieldset>
                     <div class="form-group">