Ghost/core/server/data/importer/index.js

var _            = require('lodash'),
    Promise      = require('bluebird'),
    sequence     = require('../../utils/sequence'),
    pipeline     = require('../../utils/pipeline'),
    fs           = require('fs-extra'),
    path         = require('path'),
    os           = require('os'),
    glob         = require('glob'),
    uuid         = require('node-uuid'),
    extract      = require('extract-zip'),
    errors       = require('../../errors'),
    ImageHandler    = require('./handlers/image'),
    JSONHandler     = require('./handlers/json'),
    MarkdownHandler = require('./handlers/markdown'),
    ImageImporter   = require('./importers/image'),
    DataImporter    = require('./importers/data'),

    // Glob levels
    ROOT_ONLY = 0,
    ROOT_OR_SINGLE_DIR = 1,
    ALL_DIRS = 2,

    defaults;

defaults = {
    extensions: ['.zip'],
    types: ['application/zip', 'application/x-zip-compressed'],
    directories: []
};

function ImportManager() {
    this.importers = [ImageImporter, DataImporter];
    this.handlers = [ImageHandler, JSONHandler, MarkdownHandler];
    // Keep track of files to cleanup at the end
    this.filesToDelete = [];
}

/**
 * A number, or a string containing a number.
 * @typedef {Object} ImportData
 * @property [Object] data
 * @property [Array] images
 */

_.extend(ImportManager.prototype, {
    /**
     * Get an array of all the file extensions for which we have handlers
     * @returns {string[]}
     */
    getExtensions: function () {
        return _.flatten(_.union(_.pluck(this.handlers, 'extensions'), defaults.extensions));
    },
    /**
     * Get an array of all the mime types for which we have handlers
     * @returns {string[]}
     */
    getTypes: function () {
        return _.flatten(_.union(_.pluck(this.handlers, 'types'), defaults.types));
    },
    /**
     * Get an array of directories for which we have handlers
     * @returns {string[]}
     */
    getDirectories: function () {
        return _.flatten(_.union(_.pluck(this.handlers, 'directories'), defaults.directories));
    },
    /**
     * Convert items into a glob string
     * @param {String[]} items
     * @returns {String}
     */
    getGlobPattern: function (items) {
        return '+(' + _.reduce(items, function (memo, ext) {
            return memo !== '' ? memo + '|'  + ext : ext;
        }, '') + ')';
    },
    /**
     * @param {String[]} extensions
     * @param {Number} level
     * @returns {String}
     */
    getExtensionGlob: function (extensions, level) {
        var prefix = level === ALL_DIRS ? '**/*' :
            (level === ROOT_OR_SINGLE_DIR ? '{*/*,*}' : '*');

        return prefix + this.getGlobPattern(extensions);
    },
    /**
     *
     * @param {String[]} directories
     * @param {Number} level
     * @returns {String}
     */
    getDirectoryGlob: function (directories, level) {
        var prefix = level === ALL_DIRS ? '**/' :
            (level === ROOT_OR_SINGLE_DIR ? '{*/,}' : '');

        return prefix + this.getGlobPattern(directories);
    },
    /**
     * Remove files after we're done (abstracted into a function for easier testing)
     * @returns {Function}
     */
    cleanUp: function () {
        var filesToDelete = this.filesToDelete;
        return function (result) {
            _.each(filesToDelete, function (fileToDelete) {
                fs.remove(fileToDelete, function (err) {
                    if (err) {
                        errors.logError(err, 'Import could not clean up file ', 'Your blog will continue to work as expected');
                    }
                });
            });

            return result;
        };
    },
    /**
     * Return true if the given file is a Zip
     * @returns Boolean
     */
    isZip: function (ext) {
        return _.contains(defaults.extensions, ext);
    },
    /**
     * Checks the content of a zip folder to see if it is valid.
     * Importable content includes any files or directories which the handlers can process
     * Importable content must be found either in the root, or inside one base directory
     *
     * @param {String} directory
     * @returns {Promise}
     */
    isValidZip: function (directory) {
        // Globs match content in the root or inside a single directory
        var extMatchesBase = glob.sync(
                this.getExtensionGlob(this.getExtensions(), ROOT_OR_SINGLE_DIR), {cwd: directory}
            ),
            extMatchesAll = glob.sync(
                this.getExtensionGlob(this.getExtensions(), ALL_DIRS), {cwd: directory}
            ),
            dirMatches = glob.sync(
                this.getDirectoryGlob(this.getDirectories(), ROOT_OR_SINGLE_DIR), {cwd: directory}
            ),
            oldRoonMatches = glob.sync(this.getDirectoryGlob(['drafts', 'published', 'deleted'], ROOT_OR_SINGLE_DIR),
                {cwd: directory});

        // This is a temporary extra message for the old format roon export which doesn't work with Ghost
        if (oldRoonMatches.length > 0) {
            throw new errors.UnsupportedMediaTypeError(
                'Your zip file looks like an old format Roon export, please re-export your Roon blog and try again.'
            );
        }

        // If this folder contains importable files or a content or images directory
        if (extMatchesBase.length > 0 || (dirMatches.length > 0 && extMatchesAll.length > 0)) {
            return true;
        }

        if (extMatchesAll.length < 1) {
            throw new errors.UnsupportedMediaTypeError('Zip did not include any content to import.');
        }

        throw new errors.UnsupportedMediaTypeError('Invalid zip file structure.');
    },
    /**
     * Use the extract module to extract the given zip file to a temp directory & return the temp directory path
     * @param {String} filePath
     * @returns {Promise[]} Files
     */
    extractZip: function (filePath) {
        var tmpDir = path.join(os.tmpdir(), uuid.v4());
        this.filesToDelete.push(tmpDir);
        return Promise.promisify(extract)(filePath, {dir: tmpDir}).then(function () {
            return tmpDir;
        });
    },
    /**
     * Use the handler extensions to get a globbing pattern, then use that to fetch all the files from the zip which
     * are relevant to the given handler, and return them as a name and path combo
     * @param {Object} handler
     * @param {String} directory
     * @returns [] Files
     */
    getFilesFromZip: function (handler, directory) {
        var globPattern = this.getExtensionGlob(handler.extensions, ALL_DIRS);
        return _.map(glob.sync(globPattern, {cwd: directory}), function (file) {
            return {name: file, path: path.join(directory, file)};
        });
    },
    /**
     * Get the name of the single base directory if there is one, else return an empty string
     * @param {String} directory
     * @returns {Promise (String)}
     */
    getBaseDirectory: function (directory) {
        // Globs match root level only
        var extMatches = glob.sync(this.getExtensionGlob(this.getExtensions(), ROOT_ONLY), {cwd: directory}),
            dirMatches = glob.sync(this.getDirectoryGlob(this.getDirectories(), ROOT_ONLY), {cwd: directory}),
            extMatchesAll;

        // There is no base directory
        if (extMatches.length > 0 || dirMatches.length > 0) {
            return;
        }
        // There is a base directory, grab it from any ext match
        extMatchesAll = glob.sync(
            this.getExtensionGlob(this.getExtensions(), ALL_DIRS), {cwd: directory}
        );
        if (extMatchesAll.length < 1 || extMatchesAll[0].split('/') < 1) {
            throw new errors.ValidationError('Invalid zip file: base directory read failed');
        }

        return extMatchesAll[0].split('/')[0];
    },
    /**
     * Process Zip
     * Takes a reference to a zip file, extracts it, sends any relevant files from inside to the right handler, and
     * returns an object in the importData format: {data: {}, images: []}
     * The data key contains JSON representing any data that should be imported
     * The image key contains references to images that will be stored (and where they will be stored)
     * @param {File} file
     * @returns {Promise(ImportData)}
     */
    processZip: function (file) {
        var self = this;

        return this.extractZip(file.path).then(function (zipDirectory) {
            var ops = [],
                importData = {},
                baseDir;

            self.isValidZip(zipDirectory);
            baseDir = self.getBaseDirectory(zipDirectory);

            _.each(self.handlers, function (handler) {
                if (importData.hasOwnProperty(handler.type)) {
                    // This limitation is here to reduce the complexity of the importer for now
                    return Promise.reject(new errors.UnsupportedMediaTypeError(
                        'Zip file contains multiple data formats. Please split up and import separately.'
                    ));
                }

                var files = self.getFilesFromZip(handler, zipDirectory);

                if (files.length > 0) {
                    ops.push(function () {
                        return handler.loadFile(files, baseDir).then(function (data) {
                            importData[handler.type] = data;
                        });
                    });
                }
            });

            if (ops.length === 0) {
                return Promise.reject(new errors.UnsupportedMediaTypeError(
                    'Zip did not include any content to import.'
                ));
            }

            return sequence(ops).then(function () {
                return importData;
            });
        });
    },
    /**
     * Process File
     * Takes a reference to a single file, sends it to the relevant handler to be loaded and returns an object in the
     * importData format: {data: {}, images: []}
     * The data key contains JSON representing any data that should be imported
     * The image key contains references to images that will be stored (and where they will be stored)
     * @param {File} file
     * @returns {Promise(ImportData)}
     */
    processFile: function (file, ext) {
        var fileHandler = _.find(this.handlers, function (handler) {
            return _.contains(handler.extensions, ext);
        });

        return fileHandler.loadFile([_.pick(file, 'name', 'path')]).then(function (loadedData) {
            // normalize the returned data
            var importData = {};
            importData[fileHandler.type] = loadedData;
            return importData;
        });
    },
    /**
     * Import Step 1:
     * Load the given file into usable importData in the format: {data: {}, images: []}, regardless of
     * whether the file is a single importable file like a JSON file, or a zip file containing loads of files.
     * @param {File} file
     * @returns {Promise}
     */
    loadFile: function (file) {
        var self = this,
            ext = path.extname(file.name).toLowerCase();

        this.filesToDelete.push(file.path);

        return this.isZip(ext) ? self.processZip(file) : self.processFile(file, ext);
    },
    /**
     * Import Step 2:
     * Pass the prepared importData through the preProcess function of the various importers, so that the importers can
     * make any adjustments to the data based on relationships between it
     * @param {ImportData} importData
     * @returns {Promise(ImportData)}
     */
    preProcess: function (importData) {
        var ops = [];
        _.each(this.importers, function (importer) {
            ops.push(function () {
                return importer.preProcess(importData);
            });
        });

        return pipeline(ops);
    },
    /**
     * Import Step 3:
     * Each importer gets passed the data from importData which has the key matching its type - i.e. it only gets the
     * data that it should import. Each importer then handles actually importing that data into Ghost
     * @param {ImportData} importData
     * @returns {Promise(ImportData)}
     */
    doImport: function (importData) {
        var ops = [];
        _.each(this.importers, function (importer) {
            if (importData.hasOwnProperty(importer.type)) {
                ops.push(function () {
                    return importer.doImport(importData[importer.type]);
                });
            }
        });

        return sequence(ops).then(function (importResult) {
            return importResult;
        });
    },
    /**
     * Import Step 4:
     * Report on what was imported, currently a no-op
     * @param {ImportData} importData
     * @returns {Promise(ImportData)}
     */
    generateReport: function (importData) {
        return Promise.resolve(importData);
    },
    /**
     * Import From File
     * The main method of the ImportManager, call this to kick everything off!
     * @param {File} file
     * @returns {Promise}
     */
    importFromFile: function (file) {
        var self = this;

        // Step 1: Handle converting the file to usable data
        return this.loadFile(file).then(function (importData) {
            // Step 2: Let the importers pre-process the data
            return self.preProcess(importData);
        }).then(function (importData) {
            // Step 3: Actually do the import
            // @TODO: It would be cool to have some sort of dry run flag here
            return self.doImport(importData);
        }).then(function (importData) {
            // Step 4: Report on the import
            return self.generateReport(importData)
                // Step 5: Cleanup any files
                .finally(self.cleanUp());
        });
    }
});

module.exports = new ImportManager();