Serve default robots.txt

closes #2062
- Server robots.txt from theme if available
- Serve default robots.txt from /core/shared/ otherwise
- Added tests for default robots.txt

core/server/middleware/index.js

@@ -194,6 +194,41 @@ function checkSSL(req, res, next) {
     next();
 }
 
+// ### Robots Middleware
+// Handle requests to robots.txt and cache file
+function robots() {
+    var content, // file cache
+        filePath = path.join(config().paths.corePath, '/shared/robots.txt');
+
+    return function robots(req, res, next) {
+        if ('/robots.txt' === req.url) {
+            if (content) {
+                res.writeHead(200, content.headers);
+                res.end(content.body);
+            } else {
+                fs.readFile(filePath, function (err, buf) {
+                    if (err) {
+                        return next(err);
+                    }
+                    
+                    content = {
+                        headers: {
+                            'Content-Type': 'text/plain',
+                            'Content-Length': buf.length,
+                            'Cache-Control': 'public, max-age=' + ONE_YEAR_MS / 1000
+                        },
+                        body: buf
+                    };
+                    res.writeHead(200, content.headers);
+                    res.end(content.body);
+                });
+            }
+        } else {
+            next();
+        }
+    };
+}
+
 module.exports = function (server, dbHash) {
     var logging = config().logging,
         subdir = config().paths.subdir,
@@ -229,7 +264,6 @@ module.exports = function (server, dbHash) {
     // First determine whether we're serving admin or theme content
     expressServer.use(manageAdminAndTheme);
 
-
     // Admin only config
     expressServer.use(subdir + '/ghost', middleware.whenEnabled('admin', express['static'](path.join(corePath, '/client/assets'), {maxAge: ONE_YEAR_MS})));
 
@@ -242,6 +276,9 @@ module.exports = function (server, dbHash) {
     // Theme only config
     expressServer.use(subdir, middleware.whenEnabled(expressServer.get('activeTheme'), middleware.staticTheme()));
 
+    // Serve robots.txt if not found in theme
+    expressServer.use(robots());
+
     // Add in all trailing slashes
     expressServer.use(slashes(true, {headers: {'Cache-Control': 'public, max-age=' + ONE_YEAR_S}}));
 

core/test/functional/routes/frontend_test.js

@@ -282,6 +282,13 @@ describe('Frontend Routing', function () {
                 .end(doEnd(done));
         });
 
+        it('should retrieve default robots.txt', function (done) {
+            request.get('/robots.txt')
+                .expect('Cache-Control', cacheRules.year)
+                .expect(200)
+                .end(doEnd(done));
+        })
+
         // at the moment there is no image fixture to test
         // it('should retrieve image assets', function (done) {
         // request.get('/content/images/some.jpg')