Generate sitemap files

Closes #623

- Add basic init and eventing scaffold
- Add sitemap-index.xml generation
- Broke out generators to individual files, added request handler
- Add page, author and tag xml files; add index mapping
- Add SiteMapManager unit tests
- Add Generators tests
- Cache invalidation headers for sitemap-*.xml
- Redirect sitemap.xml to index and rename to sitemap-index
- Handle page convert and publish/draft changes
- Add very basic functional test for route existence
- Add cache headers to sitemap routes

core/server/api/index.js

@@ -81,7 +81,7 @@ cacheInvalidationHeader = function (req, result) {
 
             // Don't set x-cache-invalidate header for drafts
             if (hasStatusChanged || wasDeleted || wasPublishedUpdated) {
-                cacheInvalidate = '/, /page/*, /rss/, /rss/*, /tag/*, /author/*';
+                cacheInvalidate = '/, /page/*, /rss/, /rss/*, /tag/*, /author/*, /sitemap-*.xml';
                 if (id && post.slug) {
                     return config.urlForPost(settings, post).then(function (postUrl) {
                         return cacheInvalidate + ', ' + postUrl;

core/server/data/sitemap/base-generator.js

@@ -0,0 +1,174 @@
+
+var _ = require('lodash'),
+    xml = require('xml'),
+    moment = require('moment'),
+    api = require('../../api'),
+    config = require('../../config'),
+    Promise = require('bluebird'),
+    CHANGE_FREQ = 'weekly',
+    XMLNS_DECLS;
+
+// Sitemap specific xml namespace declarations that should not change
+XMLNS_DECLS = {
+    _attr: {
+        xmlns: 'http://www.sitemaps.org/schemas/sitemap/0.9',
+        'xmlns:image': 'http://www.google.com/schemas/sitemap-image/1.1'
+    }
+};
+
+function BaseSiteMapGenerator() {
+    this.lastModified = 0;
+    this.nodeLookup = {};
+    this.siteMapContent = '';
+}
+
+_.extend(BaseSiteMapGenerator.prototype, {
+    init: function () {
+        return this.refreshAll();
+    },
+
+    getData: function () {
+        return Promise.resolve([]);
+    },
+
+    refreshAll: function () {
+        var self = this;
+
+        // Load all data
+        return this.getData().then(function (data) {
+            // Generate SiteMap from data
+            return self.generateXmlFromData(data);
+        }).then(function (generatedXml) {
+            self.siteMapContent = generatedXml;
+        });
+    },
+
+    generateXmlFromData: function (data) {
+        // This has to be async because of the permalinks retrieval
+        var self = this;
+
+        // Fetch the permalinks value only once for all the urlFor calls
+        return this.getPermalinksValue().then(function (permalinks) {
+            // Create all the url elements in JSON
+            return _.map(data, function (datum) {
+                var node = self.createUrlNodeFromDatum(datum, permalinks);
+                self.updateLastModified(datum);
+                self.nodeLookup[datum.id] = node;
+
+                return node;
+            });
+        }).then(self.generateXmlFromNodes);
+    },
+
+    getPermalinksValue: function () {
+        var self = this;
+
+        if (this.permalinks) {
+            return Promise.resolve(this.permalinks);
+        }
+
+        return api.settings.read('permalinks').then(function (response) {
+            self.permalinks = response.settings[0];
+            return self.permalinks;
+        });
+    },
+
+    updatePermalinksValue: function (permalinks) {
+        this.permalinks = permalinks;
+
+        // Re-generate xml with new permalinks values
+        this.updateXmlFromNodes(_.values(this.nodeLookup));
+    },
+
+    generateXmlFromNodes: function (urlElements) {
+        var data = {
+            // Concat the elements to the _attr declaration
+            urlset: [XMLNS_DECLS].concat(urlElements)
+        };
+
+        // Return the xml
+        return xml(data, {
+            declaration: true
+        });
+    },
+
+    updateXmlFromNodes: function (urlElements) {
+        var content = this.generateXmlFromNodes(urlElements);
+
+        this.setSiteMapContent(content);
+
+        return content;
+    },
+
+    addUrl: function (datum) {
+        var self = this;
+        return this.getPermalinksValue().then(function (permalinks) {
+            var node = self.createUrlNodeFromDatum(datum, permalinks);
+            self.updateLastModified(datum);
+            self.nodeLookup[datum.id] = node;
+
+            return self.updateXmlFromNodes(_.values(self.nodeLookup));
+        });
+    },
+
+    removeUrl: function (datum) {
+        var lookup = this.nodeLookup;
+        delete lookup[datum.id];
+
+        this.lastModified = Date.now();
+
+        return this.updateXmlFromNodes(_.values(lookup));
+    },
+
+    updateUrl: function (datum) {
+        var self = this;
+        return this.getPermalinksValue().then(function (permalinks) {
+            var node = self.createUrlNodeFromDatum(datum, permalinks);
+            self.updateLastModified(datum);
+            // TODO: Check if the node values changed, and if not don't regenerate
+            self.nodeLookup[datum.id] = node;
+
+            return self.updateXmlFromNodes(_.values(self.nodeLookup));
+        });
+    },
+
+    getUrlForDatum: function () {
+        return config.urlFor('home', true);
+    },
+
+    getUrlForImage: function (image) {
+        return config.urlFor('image', {image: image}, true);
+    },
+
+    getPriorityForDatum: function () {
+        return 1.0;
+    },
+
+    createUrlNodeFromDatum: function (datum, permalinks) {
+        var url = this.getUrlForDatum(datum, permalinks),
+            priority = this.getPriorityForDatum(datum);
+
+        return {
+            url: [
+                {loc: url},
+                {lastmod: moment(datum.updated_at || datum.published_at || datum.created_at).toISOString()},
+                {changefreq: CHANGE_FREQ},
+                {priority: priority}
+            ]
+        };
+    },
+
+    setSiteMapContent: function (content) {
+        this.siteMapContent = content;
+    },
+
+    updateLastModified: function (datum) {
+        var lastModified = datum.updated_at || datum.published_at || datum.created_at;
+
+        if (lastModified > this.lastModified) {
+            this.lastModified = lastModified;
+        }
+    }
+});
+
+module.exports = BaseSiteMapGenerator;

core/server/data/sitemap/handler.js

@@ -0,0 +1,44 @@
+var _ = require('lodash'),
+    utils = require('../utils'),
+    sitemap = require('./index');
+
+// Responsible for handling requests for sitemap files
+module.exports = function (blogApp) {
+    var resourceTypes = ['posts', 'authors', 'tags', 'pages'],
+        verifyResourceType = function (req, res, next) {
+            if (!_.contains(resourceTypes, req.param('resource'))) {
+                return res.send(404);
+            }
+
+            next();
+        },
+        getResourceSiteMapXml = function (type, page) {
+            return sitemap.getSiteMapXml(type, page);
+        };
+
+    // Redirect normal sitemap.xml requests to sitemap-index.xml
+    blogApp.get('/sitemap.xml', function (req, res) {
+        res.set({'Cache-Control': 'public, max-age=' + utils.ONE_YEAR_S});
+        res.redirect(301, '/sitemap-index.xml');
+    });
+
+    blogApp.get('/sitemap-index.xml', function (req, res) {
+        res.set({
+            'Cache-Control': 'public, max-age=' + utils.ONE_HOUR_S,
+            'Content-Type': 'text/xml'
+        });
+        res.send(sitemap.getIndexXml());
+    });
+
+    blogApp.get('/sitemap-:resource.xml', verifyResourceType, function (req, res) {
+        var type = req.param('resource'),
+            page = 1,
+            siteMapXml = getResourceSiteMapXml(type, page);
+
+        res.set({
+            'Cache-Control': 'public, max-age=' + utils.ONE_HOUR_S,
+            'Content-Type': 'text/xml'
+        });
+        res.send(siteMapXml);
+    });
+};

core/server/data/sitemap/index-generator.js

@@ -0,0 +1,54 @@
+var _ = require('lodash'),
+    xml = require('xml'),
+    moment = require('moment'),
+    config = require('../../config'),
+    RESOURCES,
+    XMLNS_DECLS;
+
+RESOURCES = ['pages', 'posts', 'authors', 'tags'];
+
+XMLNS_DECLS = {
+    _attr: {
+        xmlns: 'http://www.sitemaps.org/schemas/sitemap/0.9'
+    }
+};
+
+function SiteMapIndexGenerator(opts) {
+    // Grab the other site map generators from the options
+    _.extend(this, _.pick(opts, RESOURCES));
+}
+
+_.extend(SiteMapIndexGenerator.prototype, {
+    getIndexXml: function () {
+        var urlElements = this.generateSiteMapUrlElements(),
+            data = {
+                // Concat the elements to the _attr declaration
+                sitemapindex: [XMLNS_DECLS].concat(urlElements)
+            };
+
+        // Return the xml
+        return xml(data, {
+            declaration: true
+        });
+    },
+
+    generateSiteMapUrlElements: function () {
+        var self = this;
+
+        return _.map(RESOURCES, function (resourceType) {
+            var url = config.urlFor({
+                    relativeUrl: '/sitemap-' + resourceType + '.xml'
+                }, true),
+                lastModified = self[resourceType].lastModified;
+
+            return {
+                sitemap: [
+                    {loc: url},
+                    {lastmod: moment(lastModified).toISOString()}
+                ]
+            };
+        });
+    }
+});
+
+module.exports = SiteMapIndexGenerator;

core/server/data/sitemap/index.js

@@ -0,0 +1,4 @@
+
+var SiteMapManager = require('./manager');
+
+module.exports = new SiteMapManager();

core/server/data/sitemap/manager.js

@@ -0,0 +1,220 @@
+
+var _ = require('lodash'),
+    Promise = require('bluebird'),
+    IndexMapGenerator = require('./index-generator'),
+    PagesMapGenerator = require('./page-generator'),
+    PostsMapGenerator = require('./post-generator'),
+    UsersMapGenerator = require('./user-generator'),
+    TagsMapGenerator = require('./tag-generator'),
+    SiteMapManager;
+
+SiteMapManager = function (opts) {
+    opts = opts || {};
+
+    this.initialized = false;
+
+    this.pages = opts.pages || this.createPagesGenerator(opts);
+    this.posts = opts.posts || this.createPostsGenerator(opts);
+    this.authors = opts.authors || this.createUsersGenerator(opts);
+    this.tags = opts.tags || this.createTagsGenerator(opts);
+
+    this.index = opts.index || this.createIndexGenerator(opts);
+};
+
+_.extend(SiteMapManager.prototype, {
+    createIndexGenerator: function () {
+        return new IndexMapGenerator(_.pick(this, 'pages', 'posts', 'authors', 'tags'));
+    },
+
+    createPagesGenerator: function (opts) {
+        return new PagesMapGenerator(opts);
+    },
+
+    createPostsGenerator: function (opts) {
+        return new PostsMapGenerator(opts);
+    },
+
+    createUsersGenerator: function (opts) {
+        return new UsersMapGenerator(opts);
+    },
+
+    createTagsGenerator: function (opts) {
+        return new TagsMapGenerator(opts);
+    },
+
+    init: function () {
+        var self = this,
+            initOps = [
+                this.pages.init(),
+                this.posts.init(),
+                this.authors.init(),
+                this.tags.init()
+            ];
+
+        return Promise.all(initOps).then(function () {
+            self.initialized = true;
+        });
+    },
+
+    getIndexXml: function () {
+        if (!this.initialized) {
+            return '';
+        }
+
+        return this.index.getIndexXml();
+    },
+
+    getSiteMapXml: function (type) {
+        if (!this.initialized || !this[type]) {
+            return null;
+        }
+
+        return this[type].siteMapContent;
+    },
+
+    pageAdded: function (page) {
+        if (!this.initialized) {
+            return;
+        }
+
+        if (page.get('status') !== 'published') {
+            return;
+        }
+
+        this.pages.addUrl(page.toJSON());
+    },
+
+    pageEdited: function (page) {
+        if (!this.initialized) {
+            return;
+        }
+
+        var pageData = page.toJSON(),
+            wasPublished = page.updated('status') === 'published',
+            isPublished = pageData.status === 'published';
+
+        // Published status hasn't changed and it's published
+        if (isPublished === wasPublished && isPublished) {
+            this.pages.updateUrl(pageData);
+        } else if (!isPublished && wasPublished) {
+            // Handle page going from published to draft
+            this.pageDeleted(page);
+        } else if (isPublished && !wasPublished) {
+            // ... and draft to published
+            this.pageAdded(page);
+        }
+    },
+
+    pageDeleted: function (page) {
+        if (!this.initialized) {
+            return;
+        }
+
+        this.pages.removeUrl(page.toJSON());
+    },
+
+    postAdded: function (post) {
+        if (!this.initialized) {
+            return;
+        }
+
+        this.posts.addUrl(post.toJSON());
+    },
+
+    postEdited: function (post) {
+        if (!this.initialized) {
+            return;
+        }
+
+        var postData = post.toJSON(),
+            wasPublished = post.updated('status') === 'published',
+            isPublished = postData.status === 'published';
+
+        // Published status hasn't changed and it's published
+        if (isPublished === wasPublished && isPublished) {
+            this.posts.updateUrl(postData);
+        } else if (!isPublished && wasPublished) {
+            // Handle post going from published to draft
+            this.postDeleted(post);
+        } else if (isPublished && !wasPublished) {
+            // ... and draft to published
+            this.postAdded(post);
+        }
+    },
+
+    postDeleted: function (post) {
+        if (!this.initialized) {
+            return;
+        }
+
+        this.posts.removeUrl(post.toJSON());
+    },
+
+    userAdded: function (user) {
+        if (!this.initialized) {
+            return;
+        }
+
+        this.authors.addUrl(user.toJSON());
+    },
+
+    userEdited: function (user) {
+        if (!this.initialized) {
+            return;
+        }
+
+        var userData = user.toJSON();
+
+        this.authors.updateUrl(userData);
+    },
+
+    userDeleted: function (user) {
+        if (!this.initialized) {
+            return;
+        }
+
+        this.authors.removeUrl(user.toJSON());
+    },
+
+    tagAdded: function (tag) {
+        if (!this.initialized) {
+            return;
+        }
+
+        this.tags.addUrl(tag.toJSON());
+    },
+
+    tagEdited: function (tag) {
+        if (!this.initialized) {
+            return;
+        }
+
+        this.tags.updateUrl(tag.toJSON());
+    },
+
+    tagDeleted: function (tag) {
+        if (!this.initialized) {
+            return;
+        }
+
+        this.tags.removeUrl(tag.toJSON());
+    },
+
+    // TODO: Call this from settings model when it's changed
+    permalinksUpdated: function (permalinks) {
+        if (!this.initialized) {
+            return;
+        }
+
+        this.posts.updatePermalinksValue(permalinks.toJSON ? permalinks.toJSON() : permalinks);
+    },
+
+    _refreshAllPosts: _.throttle(function () {
+        this.posts.refreshAllPosts();
+    }, 3000, {
+        leading: false,
+        trailing: true
+    })
+});
+
+module.exports = SiteMapManager;

core/server/data/sitemap/page-generator.js

@@ -0,0 +1,75 @@
+var _ = require('lodash'),
+    path = require('path'),
+    api = require('../../api'),
+    BaseMapGenerator = require('./base-generator'),
+    config = require('../../config');
+
+// A class responsible for generating a sitemap from posts and keeping it updated
+function PageMapGenerator(opts) {
+    _.extend(this, _.defaults(opts || {}, PageMapGenerator.Defaults));
+
+    BaseMapGenerator.apply(this, arguments);
+}
+
+PageMapGenerator.Defaults = {
+    // TODO?
+};
+
+// Inherit from the base generator class
+_.extend(PageMapGenerator.prototype, BaseMapGenerator.prototype);
+
+_.extend(PageMapGenerator.prototype, {
+    getData: function () {
+        return api.posts.browse({
+            context: {
+                internal: true
+            },
+            status: 'published',
+            staticPages: true
+        }).then(function (resp) {
+            var homePage = {
+                    id: 0,
+                    name: 'home'
+                };
+            return [homePage].concat(resp.posts);
+        });
+    },
+
+    getUrlForDatum: function (post, permalinks) {
+        if (post.id === 0 && !_.isEmpty(post.name)) {
+            return config.urlFor(post.name, true);
+        }
+
+        return config.urlFor('post', {post: post, permalinks: permalinks}, true);
+    },
+
+    getPriorityForDatum: function (post) {
+        // TODO: We could influence this with priority or meta information
+        return post && post.name === 'home' ? 1.0 : 0.8;
+    },
+
+    createUrlNodeFromDatum: function (datum) {
+        var orig = BaseMapGenerator.prototype.createUrlNodeFromDatum.apply(this, arguments),
+            imageUrl,
+            imageEl;
+
+        // Check for image and add it
+        if (datum.image) {
+            // Grab the image url
+            imageUrl = this.getUrlForImage(datum.image);
+            // Create the weird xml node syntax structure that is expected
+            imageEl = [
+                {'image:loc': imageUrl},
+                {'image:caption': path.basename(imageUrl)}
+            ];
+            // Add the node to the url xml node
+            orig.url.push({
+                'image:image': imageEl
+            });
+        }
+
+        return orig;
+    }
+});
+
+module.exports = PageMapGenerator;

core/server/data/sitemap/post-generator.js

@@ -0,0 +1,67 @@
+var _ = require('lodash'),
+    path = require('path'),
+    api = require('../../api'),
+    BaseMapGenerator = require('./base-generator'),
+    config = require('../../config');
+
+// A class responsible for generating a sitemap from posts and keeping it updated
+function PostMapGenerator(opts) {
+    _.extend(this, _.defaults(opts || {}, PostMapGenerator.Defaults));
+
+    BaseMapGenerator.apply(this, arguments);
+}
+
+PostMapGenerator.Defaults = {
+    // TODO?
+};
+
+// Inherit from the base generator class
+_.extend(PostMapGenerator.prototype, BaseMapGenerator.prototype);
+
+_.extend(PostMapGenerator.prototype, {
+    getData: function () {
+        return api.posts.browse({
+            context: {
+                internal: true
+            },
+            status: 'published',
+            staticPages: false
+        }).then(function (resp) {
+            return resp.posts;
+        });
+    },
+
+    getUrlForDatum: function (post, permalinks) {
+        return config.urlFor('post', {post: post, permalinks: permalinks}, true);
+    },
+
+    getPriorityForDatum: function () {
+        // TODO: We could influence this with meta information
+        return 0.8;
+    },
+
+    createUrlNodeFromDatum: function (datum) {
+        var orig = BaseMapGenerator.prototype.createUrlNodeFromDatum.apply(this, arguments),
+            imageUrl,
+            imageEl;
+
+        // Check for image and add it
+        if (datum.image) {
+            // Grab the image url
+            imageUrl = this.getUrlForImage(datum.image);
+            // Create the weird xml node syntax structure that is expected
+            imageEl = [
+                {'image:loc': imageUrl},
+                {'image:caption': path.basename(imageUrl)}
+            ];
+            // Add the node to the url xml node
+            orig.url.push({
+                'image:image': imageEl
+            });
+        }
+
+        return orig;
+    }
+});
+
+module.exports = PostMapGenerator;

core/server/data/sitemap/tag-generator.js

@@ -0,0 +1,41 @@
+var _ = require('lodash'),
+    api = require('../../api'),
+    BaseMapGenerator = require('./base-generator'),
+    config = require('../../config');
+
+// A class responsible for generating a sitemap from posts and keeping it updated
+function TagsMapGenerator(opts) {
+    _.extend(this, _.defaults(opts || {}, TagsMapGenerator.Defaults));
+
+    BaseMapGenerator.apply(this, arguments);
+}
+
+TagsMapGenerator.Defaults = {
+    // TODO?
+};
+
+// Inherit from the base generator class
+_.extend(TagsMapGenerator.prototype, BaseMapGenerator.prototype);
+
+_.extend(TagsMapGenerator.prototype, {
+    getData: function () {
+        return api.tags.browse({
+            context: {
+                internal: true
+            }
+        }).then(function (resp) {
+            return resp.tags;
+        });
+    },
+
+    getUrlForDatum: function (tag, permalinks) {
+        return config.urlFor('tag', {tag: tag, permalinks: permalinks}, true);
+    },
+
+    getPriorityForDatum: function () {
+        // TODO: We could influence this with meta information
+        return 0.6;
+    }
+});
+
+module.exports = TagsMapGenerator;

core/server/data/sitemap/user-generator.js

@@ -0,0 +1,65 @@
+var _ = require('lodash'),
+    path = require('path'),
+    api = require('../../api'),
+    BaseMapGenerator = require('./base-generator'),
+    config = require('../../config');
+
+// A class responsible for generating a sitemap from posts and keeping it updated
+function UserMapGenerator(opts) {
+    _.extend(this, _.defaults(opts || {}, UserMapGenerator.Defaults));
+
+    BaseMapGenerator.apply(this, arguments);
+}
+
+UserMapGenerator.Defaults = {
+    // TODO?
+};
+
+// Inherit from the base generator class
+_.extend(UserMapGenerator.prototype, BaseMapGenerator.prototype);
+
+_.extend(UserMapGenerator.prototype, {
+    getData: function () {
+        return api.users.browse({
+            context: {
+                internal: true
+            }
+        }).then(function (resp) {
+            return resp.users;
+        });
+    },
+
+    getUrlForDatum: function (user, permalinks) {
+        return config.urlFor('author', {author: user, permalinks: permalinks}, true);
+    },
+
+    getPriorityForDatum: function () {
+        // TODO: We could influence this with meta information
+        return 0.6;
+    },
+
+    createUrlNodeFromDatum: function (datum) {
+        var orig = BaseMapGenerator.prototype.createUrlNodeFromDatum.apply(this, arguments),
+            imageUrl,
+            imageEl;
+
+        // Check for image and add it
+        if (datum.image) {
+            // Grab the image url
+            imageUrl = this.getUrlForImage(datum.image);
+            // Create the weird xml node syntax structure that is expected
+            imageEl = [
+                {'image:loc': imageUrl},
+                {'image:caption': path.basename(imageUrl)}
+            ];
+            // Add the node to the url xml node
+            orig.url.push({
+                'image:image': imageEl
+            });
+        }
+
+        return orig;
+    }
+});
+
+module.exports = UserMapGenerator;

core/server/index.js

@@ -17,6 +17,7 @@ var express     = require('express'),
     models      = require('./models'),
     permissions = require('./permissions'),
     apps        = require('./apps'),
+    sitemap     = require('./data/sitemap'),
     GhostServer = require('./ghost-server'),
 
 // Variables
@@ -166,7 +167,9 @@ function init(options) {
             // Initialize mail
             mailer.init(),
             // Initialize apps
-            apps.init()
+            apps.init(),
+            // Initialize sitemaps
+            sitemap.init()
         );
     }).then(function () {
         var adminHbs = hbs.create();

core/server/middleware/index.js

@@ -23,6 +23,7 @@ var api            = require('../api'),
     oauth2orize    = require('oauth2orize'),
     authStrategies = require('./auth-strategies'),
     utils          = require('../utils'),
+    sitemapHandler = require('../data/sitemap/handler'),
 
     blogApp,
     setupMiddleware;
@@ -291,6 +292,9 @@ setupMiddleware = function (blogAppInstance, adminApp) {
     // Serve robots.txt if not found in theme
     blogApp.use(serveSharedFile('robots.txt', 'text/plain', utils.ONE_HOUR_S));
 
+    // site map
+    sitemapHandler(blogApp);
+
     // Add in all trailing slashes, properly include the subdir path
     // in the redirect.
     blogApp.use(slashes(true, {

core/server/models/post.js

@@ -8,6 +8,7 @@ var _              = require('lodash'),
     converter      = new Showdown.converter({extensions: [ghostgfm]}),
     ghostBookshelf = require('./base'),
     xmlrpc         = require('../xmlrpc'),
+    sitemap        = require('../data/sitemap'),
 
     Post,
     Posts;
@@ -34,6 +35,43 @@ Post = ghostBookshelf.Model.extend({
             }
             return self.updateTags(model, attributes, options);
         });
+
+        this.on('created', function (model) {
+            var isPage = !!model.get('page');
+            if (isPage) {
+                sitemap.pageAdded(model);
+            } else {
+                sitemap.postAdded(model);
+            }
+        });
+        this.on('updated', function (model) {
+            var isPage = !!model.get('page'),
+                wasPage = !!model.updated('page');
+
+            if (isPage && wasPage) {
+                // Page value didn't change, remains a page
+                sitemap.pageEdited(model);
+            } else if (!isPage && !wasPage) {
+                // Remains a Post
+                sitemap.postEdited(model);
+            } else if (isPage && !wasPage) {
+                // Switched from Post to Page
+                sitemap.postDeleted(model);
+                sitemap.pageAdded(model);
+            } else if (!isPage && wasPage) {
+                // Switched from Page to Post
+                sitemap.pageDeleted(model);
+                sitemap.postAdded(model);
+            }
+        });
+        this.on('destroyed', function (model) {
+            var isPage = !!model.get('page');
+            if (isPage) {
+                sitemap.pageDeleted(model);
+            } else {
+                sitemap.postDeleted(model);
+            }
+        });
     },
 
     saving: function (newPage, attr, options) {

core/server/models/tag.js

@@ -1,6 +1,7 @@
 var _              = require('lodash'),
     errors         = require('../errors'),
     ghostBookshelf = require('./base'),
+    sitemap        = require('../data/sitemap'),
 
     Tag,
     Tags;
@@ -9,6 +10,20 @@ Tag = ghostBookshelf.Model.extend({
 
     tableName: 'tags',
 
+    initialize: function () {
+        ghostBookshelf.Model.prototype.initialize.apply(this, arguments);
+
+        this.on('created', function (model) {
+            sitemap.tagAdded(model);
+        });
+        this.on('updated', function (model) {
+            sitemap.tagEdited(model);
+        });
+        this.on('destroyed', function (model) {
+            sitemap.tagDeleted(model);
+        });
+    },
+
     saving: function (newPage, attr, options) {
          /*jshint unused:false*/
 

core/server/models/user.js

@@ -8,6 +8,7 @@ var _              = require('lodash'),
     request        = require('request'),
     validation     = require('../data/validation'),
     config         = require('../config'),
+    sitemap        = require('../data/sitemap'),
 
     bcryptGenSalt  = Promise.promisify(bcrypt.genSalt),
     bcryptHash     = Promise.promisify(bcrypt.hash),
@@ -42,6 +43,20 @@ User = ghostBookshelf.Model.extend({
 
     tableName: 'users',
 
+    initialize: function () {
+        ghostBookshelf.Model.prototype.initialize.apply(this, arguments);
+
+        this.on('created', function (model) {
+            sitemap.userAdded(model);
+        });
+        this.on('updated', function (model) {
+            sitemap.userEdited(model);
+        });
+        this.on('destroyed', function (model) {
+            sitemap.userDeleted(model);
+        });
+    },
+
     saving: function (newPage, attr, options) {
         /*jshint unused:false*/
 

core/shared/robots.txt

@@ -1,2 +1,3 @@
 User-agent: *
-Disallow: /ghost/
+Sitemap: /sitemap-index.xml
+Disallow: /ghost/

core/test/functional/routes/api/posts_test.js

@@ -363,7 +363,7 @@ describe('Post API', function () {
                             var publishedPost = res.body;
                             _.has(res.headers, 'x-cache-invalidate').should.equal(true);
                             res.headers['x-cache-invalidate'].should.eql(
-                                '/, /page/*, /rss/, /rss/*, /tag/*, /author/*, /' + publishedPost.posts[0].slug + '/'
+                                '/, /page/*, /rss/, /rss/*, /tag/*, /author/*, /sitemap-*.xml, /' + publishedPost.posts[0].slug + '/'
                             );
 
                             publishedPost.should.exist;
@@ -782,7 +782,7 @@ describe('Post API', function () {
                     jsonResponse.should.exist;
                     jsonResponse.posts.should.exist;
                     res.headers['x-cache-invalidate'].should.eql(
-                        '/, /page/*, /rss/, /rss/*, /tag/*, /author/*, /' + jsonResponse.posts[0].slug + '/'
+                        '/, /page/*, /rss/, /rss/*, /tag/*, /author/*, /sitemap-*.xml, /' + jsonResponse.posts[0].slug + '/'
                     );
                     testUtils.API.checkResponse(jsonResponse.posts[0], 'post');
                     jsonResponse.posts[0].id.should.eql(deletePostId);

core/test/functional/routes/frontend_test.js

@@ -925,4 +925,44 @@ describe('Frontend Routing', function () {
                 });
         });
     });
+
+    describe('Site Map', function () {
+        before(function (done) {
+            testUtils.initData().then(function () {
+                return testUtils.fixtures.insertPosts();
+            }).then(function () {
+                done();
+            }).catch(done);
+        });
+
+        it('should redirect for /sitemap.xml', function (done) {
+            request.get('/sitemap.xml')
+                .expect(301)
+                .expect('location', /sitemap-index.xml/)
+                .end(doEnd(done));
+        });
+
+        it('should serve sitemap-index.xml', function (done) {
+            request.get('/sitemap-index.xml')
+                .expect(200)
+                .expect('Content-Type', 'text/xml; charset=utf-8')
+                .end(doEnd(done));
+        });
+
+        it('should serve sitemap-posts.xml', function (done) {
+            request.get('/sitemap-posts.xml')
+                .expect(200)
+                .expect('Content-Type', 'text/xml; charset=utf-8')
+                .end(doEnd(done));
+        });
+
+        it('should serve sitemap-pages.xml', function (done) {
+            request.get('/sitemap-posts.xml')
+                .expect(200)
+                .expect('Content-Type', 'text/xml; charset=utf-8')
+                .end(doEnd(done));
+        });
+
+        // TODO: Other pages and verify content
+    });
 });

core/test/integration/sitemap_spec.js

@@ -0,0 +1,469 @@
+/*globals describe, before, afterEach, it */
+/*jshint expr:true*/
+var testUtils   = require('../utils/index'),
+    _           = require('lodash'),
+    should      = require('should'),
+    sinon       = require('sinon'),
+    Promise     = require('bluebird'),
+    validator   = require('validator'),
+
+    // Stuff we are testing
+    SiteMapManager = require('../../server/data/sitemap/manager'),
+    BaseGenerator = require('../../server/data/sitemap/base-generator'),
+    PostGenerator = require('../../server/data/sitemap/post-generator'),
+    PageGenerator = require('../../server/data/sitemap/page-generator'),
+    TagGenerator = require('../../server/data/sitemap/tag-generator'),
+    UserGenerator = require('../../server/data/sitemap/user-generator'),
+
+    sandbox = sinon.sandbox.create();
+
+describe('Sitemap', function () {
+    var makeStubManager = function () {
+        return new SiteMapManager({
+            pages: {
+                init: sandbox.stub().returns(Promise.resolve()),
+                addUrl: sandbox.stub(),
+                removeUrl: sandbox.stub(),
+                updateUrl: sandbox.stub()
+            },
+            posts: {
+                init: sandbox.stub().returns(Promise.resolve()),
+                addUrl: sandbox.stub(),
+                removeUrl: sandbox.stub(),
+                updateUrl: sandbox.stub()
+            },
+            authors: {
+                init: sandbox.stub().returns(Promise.resolve()),
+                addUrl: sandbox.stub(),
+                removeUrl: sandbox.stub(),
+                updateUrl: sandbox.stub()
+            },
+            tags: {
+                init: sandbox.stub().returns(Promise.resolve()),
+                addUrl: sandbox.stub(),
+                removeUrl: sandbox.stub(),
+                updateUrl: sandbox.stub()
+            },
+            index: {
+                init: sandbox.stub().returns(Promise.resolve()),
+                addUrl: sandbox.stub(),
+                removeUrl: sandbox.stub(),
+                updateUrl: sandbox.stub()
+            }
+        });
+    };
+
+    before(testUtils.teardown);
+    afterEach(testUtils.teardown);
+    afterEach(function () {
+        sandbox.restore();
+    });
+
+    describe('SiteMapManager', function () {
+        should.exist(SiteMapManager);
+
+        it('can create a SiteMapManager instance', function () {
+            var manager = makeStubManager();
+
+            should.exist(manager);
+        });
+
+        it('can initialize', function (done) {
+            var manager = makeStubManager();
+
+            manager.initialized.should.equal(false);
+
+            manager.init().then(function () {
+                manager.posts.init.called.should.equal(true);
+                manager.pages.init.called.should.equal(true);
+                manager.authors.init.called.should.equal(true);
+                manager.tags.init.called.should.equal(true);
+
+                manager.initialized.should.equal(true);
+
+                done();
+            }).catch(done);
+        });
+
+        it('responds to calls before being initialized', function () {
+            var manager = makeStubManager();
+
+            manager.initialized.should.equal(false);
+
+            manager.getIndexXml();
+            manager.getSiteMapXml();
+            manager.pageAdded();
+            manager.pages.addUrl.called.should.equal(false);
+            manager.pageEdited();
+            manager.pageDeleted();
+            manager.postAdded();
+            manager.pages.addUrl.called.should.equal(false);
+            manager.postEdited();
+            manager.postDeleted();
+            manager.userAdded();
+            manager.pages.addUrl.called.should.equal(false);
+            manager.userEdited();
+            manager.userDeleted();
+            manager.tagAdded();
+            manager.pages.addUrl.called.should.equal(false);
+            manager.tagEdited();
+            manager.tagDeleted();
+            manager.permalinksUpdated();
+
+            manager.initialized.should.equal(false);
+        });
+
+        it('updates page site map', function (done) {
+            var manager = makeStubManager(),
+                fake = {
+                    toJSON: sandbox.stub().returns({
+                        status: 'published'
+                    }),
+                    get: sandbox.stub().returns('published'),
+                    updated: sandbox.stub().returns('published')
+                };
+
+            manager.init().then(function () {
+                manager.pageAdded(fake);
+                manager.pages.addUrl.called.should.equal(true);
+                manager.pageEdited(fake);
+                manager.pages.updateUrl.called.should.equal(true);
+                manager.pageDeleted(fake);
+                manager.pages.removeUrl.called.should.equal(true);
+
+                done();
+            }).catch(done);
+        });
+
+        it('adds pages that were published', function (done) {
+            var manager = makeStubManager(),
+                fake = {
+                    toJSON: sandbox.stub().returns({
+                        status: 'published'
+                    }),
+                    get: sandbox.stub().returns('published'),
+                    updated: sandbox.stub().returns('draft')
+                };
+
+            manager.init().then(function () {
+                manager.pageAdded = sandbox.stub();
+
+                manager.pageEdited(fake);
+
+                manager.pages.updateUrl.called.should.equal(false);
+                manager.pageAdded.called.should.equal(true);
+
+                done();
+            }).catch(done);
+        });
+
+        it('deletes pages that were unpublished', function (done) {
+            var manager = makeStubManager(),
+                fake = {
+                    toJSON: sandbox.stub().returns({
+                        status: 'draft'
+                    }),
+                    get: sandbox.stub().returns('draft'),
+                    updated: sandbox.stub().returns('published')
+                };
+
+            manager.init().then(function () {
+                manager.pageAdded = sandbox.stub();
+                manager.pageDeleted = sandbox.stub();
+
+                manager.pageEdited(fake);
+
+                manager.pages.updateUrl.called.should.equal(false);
+                manager.pageAdded.called.should.equal(false);
+                manager.pageDeleted.called.should.equal(true);
+
+                done();
+            }).catch(done);
+        });
+
+        it('updates post site map', function (done) {
+            var manager = makeStubManager(),
+                fake = {
+                    toJSON: sandbox.stub().returns({
+                        status: 'published'
+                    }),
+                    get: sandbox.stub().returns('published'),
+                    updated: sandbox.stub().returns('published')
+                };
+
+            manager.init().then(function () {
+                manager.postAdded(fake);
+                manager.posts.addUrl.called.should.equal(true);
+                manager.postEdited(fake);
+                manager.posts.updateUrl.called.should.equal(true);
+                manager.postDeleted(fake);
+                manager.posts.removeUrl.called.should.equal(true);
+
+                done();
+            }).catch(done);
+        });
+
+        it('adds posts that were published', function (done) {
+            var manager = makeStubManager(),
+                fake = {
+                    toJSON: sandbox.stub().returns({
+                        status: 'published'
+                    }),
+                    get: sandbox.stub().returns('published'),
+                    updated: sandbox.stub().returns('draft')
+                };
+
+            manager.init().then(function () {
+                manager.postAdded = sandbox.stub();
+
+                manager.postEdited(fake);
+
+                manager.posts.updateUrl.called.should.equal(false);
+                manager.postAdded.called.should.equal(true);
+
+                done();
+            }).catch(done);
+        });
+
+        it('deletes posts that were unpublished', function (done) {
+            var manager = makeStubManager(),
+                fake = {
+                    toJSON: sandbox.stub().returns({
+                        status: 'draft'
+                    }),
+                    get: sandbox.stub().returns('draft'),
+                    updated: sandbox.stub().returns('published')
+                };
+
+            manager.init().then(function () {
+                manager.postAdded = sandbox.stub();
+                manager.postDeleted = sandbox.stub();
+
+                manager.postEdited(fake);
+
+                manager.posts.updateUrl.called.should.equal(false);
+                manager.postAdded.called.should.equal(false);
+                manager.postDeleted.called.should.equal(true);
+
+                done();
+            }).catch(done);
+        });
+
+        it('updates authors site map', function (done) {
+            var manager = makeStubManager(),
+                fake = {
+                    toJSON: sandbox.stub().returns({})
+                };
+
+            manager.init().then(function () {
+                manager.userAdded(fake);
+                manager.authors.addUrl.called.should.equal(true);
+                manager.userEdited(fake);
+                manager.authors.updateUrl.called.should.equal(true);
+                manager.userDeleted(fake);
+                manager.authors.removeUrl.called.should.equal(true);
+
+                done();
+            }).catch(done);
+        });
+
+        it('updates tags site map', function (done) {
+            var manager = makeStubManager(),
+                fake = {
+                    toJSON: sandbox.stub().returns({})
+                };
+
+            manager.init().then(function () {
+                manager.tagAdded(fake);
+                manager.tags.addUrl.called.should.equal(true);
+                manager.tagEdited(fake);
+                manager.tags.updateUrl.called.should.equal(true);
+                manager.tagDeleted(fake);
+                manager.tags.removeUrl.called.should.equal(true);
+
+                done();
+            }).catch(done);
+        });
+    });
+
+    describe('Generators', function () {
+        var stubPermalinks = function (generator) {
+                sandbox.stub(generator, 'getPermalinksValue', function () {
+                    return Promise.resolve({
+                        id: 13,
+                        uuid: 'ac6d6bb2-0b64-4941-b5ef-e69000bb738a',
+                        key: 'permalinks',
+                        value: '/:slug/',
+                        type: 'blog'
+                    });
+                });
+
+                return generator;
+            },
+            stubUrl = function (generator) {
+                sandbox.stub(generator, 'getUrlForDatum', function (datum) {
+                    return 'http://my-ghost-blog.com/url/' + datum.id;
+                });
+                sandbox.stub(generator, 'getUrlForImage', function (image) {
+                    return 'http://my-ghost-blog.com/images/' + image;
+                });
+
+                return generator;
+            },
+            makeFakeDatum = function (id) {
+                return {
+                    id: id,
+                    created_at: (Date.UTC(2014, 11, 22, 12) - 360000) + id
+                };
+            };
+
+        describe('BaseGenerator', function () {
+            it('can initialize with empty siteMapContent', function (done) {
+                var generator = new BaseGenerator();
+
+                stubPermalinks(generator);
+
+                generator.init().then(function () {
+                    should.exist(generator.siteMapContent);
+
+                    validator.contains(generator.siteMapContent, '<loc>').should.equal(false);
+
+                    done();
+                }).catch(done);
+            });
+
+            it('can initialize with non-empty siteMapContent', function (done) {
+                var generator = new BaseGenerator();
+
+                stubPermalinks(generator);
+                stubUrl(generator);
+
+                sandbox.stub(generator, 'getData', function () {
+                    return Promise.resolve([
+                        makeFakeDatum(100),
+                        makeFakeDatum(200),
+                        makeFakeDatum(300)
+                    ]);
+                });
+
+                generator.init().then(function () {
+                    should.exist(generator.siteMapContent);
+
+                    // TODO: We should validate the contents against the XSD:
+                    // xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                    // xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"
+
+                    validator.contains(generator.siteMapContent,
+                        '<loc>http://my-ghost-blog.com/url/100</loc>').should.equal(true);
+                    validator.contains(generator.siteMapContent,
+                        '<loc>http://my-ghost-blog.com/url/200</loc>').should.equal(true);
+                    validator.contains(generator.siteMapContent,
+                        '<loc>http://my-ghost-blog.com/url/300</loc>').should.equal(true);
+
+                    done();
+                }).catch(done);
+            });
+        });
+
+        describe('PostGenerator', function () {
+            it('uses 0.8 priority for all posts', function () {
+                var generator = new PostGenerator();
+
+                generator.getPriorityForDatum({}).should.equal(0.8);
+            });
+
+            it('adds an image:image element if post has a cover image', function () {
+                var generator = new PostGenerator(),
+                    urlNode = generator.createUrlNodeFromDatum(_.extend(makeFakeDatum(100), {
+                        image: 'post-100.jpg'
+                    })),
+                    hasImage;
+
+                hasImage = _.any(urlNode.url, function (node) {
+                    return !_.isUndefined(node['image:image']);
+                });
+
+                hasImage.should.equal(true);
+            });
+
+            it('can initialize with non-empty siteMapContent', function (done) {
+                var generator = new PostGenerator();
+
+                stubPermalinks(generator);
+                stubUrl(generator);
+
+                sandbox.stub(generator, 'getData', function () {
+                    return Promise.resolve([
+                        _.extend(makeFakeDatum(100), {
+                            image: 'post-100.jpg'
+                        }),
+                        makeFakeDatum(200),
+                        _.extend(makeFakeDatum(300), {
+                            image: 'post-300.jpg'
+                        })
+                    ]);
+                });
+
+                generator.init().then(function () {
+                    should.exist(generator.siteMapContent);
+
+                    // TODO: We should validate the contents against the XSD:
+                    // xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                    // xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9
+                    // http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"
+
+                    validator.contains(generator.siteMapContent,
+                        '<loc>http://my-ghost-blog.com/url/100</loc>').should.equal(true);
+                    validator.contains(generator.siteMapContent,
+                        '<loc>http://my-ghost-blog.com/url/200</loc>').should.equal(true);
+                    validator.contains(generator.siteMapContent,
+                        '<loc>http://my-ghost-blog.com/url/300</loc>').should.equal(true);
+
+                    validator.contains(generator.siteMapContent,
+                        '<image:loc>http://my-ghost-blog.com/images/post-100.jpg</image:loc>')
+                        .should.equal(true);
+                    // This should NOT be present
+                    validator.contains(generator.siteMapContent,
+                        '<image:loc>http://my-ghost-blog.com/images/post-200.jpg</image:loc>')
+                        .should.equal(false);
+                    validator.contains(generator.siteMapContent,
+                        '<image:loc>http://my-ghost-blog.com/images/post-300.jpg</image:loc>')
+                        .should.equal(true);
+
+                    done();
+                }).catch(done);
+            });
+        });
+
+        describe('PageGenerator', function () {
+            it('uses 1 priority for home page', function () {
+                var generator = new PageGenerator();
+
+                generator.getPriorityForDatum({
+                    name: 'home'
+                }).should.equal(1);
+            });
+            it('uses 0.8 priority for static pages', function () {
+                var generator = new PageGenerator();
+
+                generator.getPriorityForDatum({}).should.equal(0.8);
+            });
+        });
+
+        describe('TagGenerator', function () {
+            it('uses 0.6 priority for all tags', function () {
+                var generator = new TagGenerator();
+
+                generator.getPriorityForDatum({}).should.equal(0.6);
+            });
+        });
+
+        describe('UserGenerator', function () {
+            it('uses 0.6 priority for author links', function () {
+                var generator = new UserGenerator();
+
+                generator.getPriorityForDatum({}).should.equal(0.6);
+            });
+        });
+    });
+});