mirror of
https://github.com/remy/inliner.git
synced 2024-10-05 14:57:27 +03:00
refactor: HTML pulling down, events, images
This commit is contained in:
Remy Sharp
parent
c940480c48
commit
16944df764
8
.jscsrc
Normal file
8
.jscsrc
Normal file
@ -0,0 +1,8 @@
|
||||
{
|
||||
"preset": "node-style-guide",
|
||||
"requireCapitalizedComments": null,
|
||||
"requireSpacesInAnonymousFunctionExpression": {
|
||||
"beforeOpeningCurlyBrace": true
|
||||
},
|
||||
"excludeFiles": ["node_modules/**"]
|
||||
}
|
||||
16
.jshintrc
Normal file
16
.jshintrc
Normal file
@ -0,0 +1,16 @@
|
||||
{
|
||||
"browser": true,
|
||||
"camelcase": true,
|
||||
"curly": true,
|
||||
"devel": true,
|
||||
"eqeqeq": true,
|
||||
"forin": true,
|
||||
"indent": 2,
|
||||
"noarg": true,
|
||||
"node": true,
|
||||
"quotmark": "single",
|
||||
"undef": true,
|
||||
"strict": false,
|
||||
"unused": true
|
||||
}
|
||||
|
||||
108
bin/inliner
108
bin/inliner
@ -1,108 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
var path = require('path'),
|
||||
fs = require('fs'),
|
||||
querystring = require('querystring'),
|
||||
util = require('util');
|
||||
|
||||
fs.realpath(__filename, function(error, script) {
|
||||
var servedir, root, port;
|
||||
if (error) throw error;
|
||||
|
||||
var Inliner = require(path.join(path.dirname(script), '../inliner')),
|
||||
program = require('commander');
|
||||
|
||||
program
|
||||
.version(Inliner.version)
|
||||
.usage('[options] http://yoursite.com')
|
||||
.option('-v, --verbose', 'echo on STDERR the progress of inlining')
|
||||
.option('-n, --nocompress', "don't compress CSS or HTML - useful for debugging")
|
||||
.option('-i, --images', "don't encode images - keeps files size small, but more requests")
|
||||
//.option('-s, --share', "create jsbin.com url for HTML page")
|
||||
//.option('-h, --help', "help - you're looking at it");
|
||||
|
||||
program.on('--help', function () {
|
||||
console.log(' Examples:');
|
||||
console.log('');
|
||||
console.log(' $ inliner -v http://twitter.com > twitter.html');
|
||||
console.log(' $ inliner -ni http://twitter.com > twitter.html');
|
||||
console.log('');
|
||||
console.log(' For more details see http://github.com/remy/inliner/');
|
||||
console.log('');
|
||||
});
|
||||
|
||||
program.parse(process.argv);
|
||||
|
||||
var options = Inliner.defaults(),
|
||||
verbose = false,
|
||||
share = false;
|
||||
|
||||
if (program.nocompress) {
|
||||
options.compressCSS = false;
|
||||
options.collapseWhitespace = false;
|
||||
}
|
||||
|
||||
if (program.share) {
|
||||
share = true;
|
||||
|
||||
// because JS Bin is a damn sight easier to debug when it's not all on one line
|
||||
options.compressCSS = false;
|
||||
options.collapseWhitespace = false;
|
||||
}
|
||||
|
||||
options.images = !program.images;
|
||||
verbose = program.verbose;
|
||||
|
||||
if (program.args.length == 0) {
|
||||
// ripped out of commander.js - should really be process.usage()
|
||||
process.stdout.write(program.helpInformation());
|
||||
program.emit('--help');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
var url = program.args[0];
|
||||
|
||||
if (fs.existsSync(url)) {
|
||||
// then it's a file
|
||||
} else if (url.indexOf('http') !== 0) {
|
||||
url = 'http://' + url;
|
||||
}
|
||||
|
||||
var inliner = new Inliner(url, options, function (html) {
|
||||
if (share) {
|
||||
// post to jsbin
|
||||
var data = querystring.stringify({
|
||||
html: html,
|
||||
javascript: '',
|
||||
format: 'plain',
|
||||
method: 'save'
|
||||
});
|
||||
|
||||
// note: when making a POST request using node, for PHP to pick it up, the content-type is crutial - I never knew that :(
|
||||
var request = Inliner.makeRequest('http://jsbin.com/save', {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/x-www-form-urlencoded', 'content-length': data.length, 'X-Requested-With' : 'XMLHttpRequest' }
|
||||
});
|
||||
request.on('response', function (res) {
|
||||
var body = '';
|
||||
res.on('data', function (chunk) {
|
||||
body += chunk;
|
||||
});
|
||||
res.on('end', function () {
|
||||
util.print(body);
|
||||
});
|
||||
});
|
||||
request.write(data);
|
||||
request.end();
|
||||
} else {
|
||||
// using util.print because console.log evalutes sprintf commands - which we don't want to do
|
||||
util.print(html);
|
||||
}
|
||||
});
|
||||
|
||||
if (verbose) {
|
||||
inliner.on('progress', function (event) {
|
||||
console.error(event);
|
||||
});
|
||||
}
|
||||
});
|
||||
81
cli/index.js
Executable file
81
cli/index.js
Executable file
@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env node
|
||||
var argv = require('minimist')(process.argv.slice(2), {
|
||||
alias: {
|
||||
V: 'version',
|
||||
h: 'help',
|
||||
d: 'debug',
|
||||
v: 'verbose',
|
||||
i: 'images',
|
||||
n: 'nocompress',
|
||||
},
|
||||
});
|
||||
|
||||
if (argv.debug) {
|
||||
require('debug').enable('inliner');
|
||||
}
|
||||
|
||||
var Inliner = require('../');
|
||||
|
||||
// checks for available update and returns an instance
|
||||
// var updateNotifier = require('update-notifier');
|
||||
// var pkg = require('../package.json');
|
||||
// var notifier = updateNotifier({ pkg: pkg });
|
||||
// if (notifier.update) {
|
||||
// // notify using the built-in convenience method
|
||||
// notifier.notify();
|
||||
// }
|
||||
|
||||
var url = argv._.shift();
|
||||
|
||||
var argvKeys = Object.keys(argv).filter(function (item) {
|
||||
return item !== '_';
|
||||
});
|
||||
|
||||
if (!url && argvKeys.length === 0 || argv.help) {
|
||||
// show USAGE!
|
||||
console.log(' Examples:');
|
||||
console.log('');
|
||||
console.log(' $ inliner -v http://twitter.com > twitter.html');
|
||||
console.log(' $ inliner -ni http://twitter.com > twitter.html');
|
||||
console.log('');
|
||||
console.log(' For more details see http://github.com/remy/inliner/');
|
||||
console.log('');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (argv.version) {
|
||||
console.log(Inliner.version);
|
||||
}
|
||||
|
||||
var options = Inliner.defaults();
|
||||
|
||||
if (argv.nocompress) {
|
||||
options.compressCSS = false;
|
||||
options.collapseWhitespace = false;
|
||||
}
|
||||
|
||||
options.images = !argv.images;
|
||||
|
||||
var inliner = new Inliner(url, argv, function result(error, html) {
|
||||
if (error) {
|
||||
var message = Inliner.errors[error.code] || error.message;
|
||||
console.error(message);
|
||||
|
||||
if (argv.debug) {
|
||||
console.error(error.stack);
|
||||
}
|
||||
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(html);
|
||||
});
|
||||
|
||||
if (argv.verbose) {
|
||||
inliner.on('progress', function progress(event) {
|
||||
console.error(event);
|
||||
});
|
||||
|
||||
inliner.on('jobs', function jobs(event) {
|
||||
console.error(event);
|
||||
});
|
||||
}
|
||||
566
inliner.js
566
inliner.js
@ -1,566 +0,0 @@
|
||||
var URL = require('url'),
|
||||
util = require('util'),
|
||||
jsmin = require('./jsmin'),
|
||||
events = require('events'),
|
||||
Buffer = require('buffer').Buffer,
|
||||
fs = require('fs'),
|
||||
path = require('path'),
|
||||
jsdom = require('jsdom'),
|
||||
uglifyjs = require('uglify-js'),
|
||||
jsp = uglifyjs.parser,
|
||||
pro = uglifyjs.uglify,
|
||||
compress = null, // import only when required - might make the command line tool easier to use
|
||||
http = {
|
||||
http: require('http'),
|
||||
https: require('https')
|
||||
};
|
||||
|
||||
function compressCSS(css) {
|
||||
return css
|
||||
.replace(/\s+/g, ' ')
|
||||
.replace(/:\s+/g, ':')
|
||||
.replace(/\/\*.*?\*\//g, '')
|
||||
.replace(/\} /g, '}')
|
||||
.replace(/ \{/g, '{')
|
||||
// .replace(/\{ /g, '{')
|
||||
.replace(/; /g, ';')
|
||||
.replace(/\n+/g, '');
|
||||
}
|
||||
|
||||
function removeComments(element) {
|
||||
if (!element || !element.childNodes) return;
|
||||
var nodes = element.childNodes,
|
||||
i = nodes.length;
|
||||
|
||||
while (i--) {
|
||||
if (nodes[i].nodeName === '#comment' && nodes[i].nodeValue.indexOf('[') !== 0) {
|
||||
element.removeChild(nodes[i]);
|
||||
}
|
||||
removeComments(nodes[i]);
|
||||
}
|
||||
}
|
||||
|
||||
function Inliner(url, options, callback) {
|
||||
var root = url,
|
||||
inliner = this;
|
||||
|
||||
this.requestCache = {};
|
||||
this.requestCachePending = {};
|
||||
|
||||
// inherit EventEmitter so that we can send events with progress
|
||||
events.EventEmitter.call(this);
|
||||
|
||||
if (typeof options == 'function') {
|
||||
callback = options;
|
||||
options = Inliner.defaults();
|
||||
} else if (options === undefined) {
|
||||
options = Inliner.defaults();
|
||||
}
|
||||
|
||||
inliner.options = options;
|
||||
|
||||
inliner.total = 1;
|
||||
inliner.todo = 1;
|
||||
inliner.emit('jobs', (inliner.total - inliner.todo) + '/' + inliner.total);
|
||||
|
||||
inliner.on('error', function (data) {
|
||||
console.error(data + ' :: ' + url);
|
||||
});
|
||||
|
||||
inliner.get(url, function (html) {
|
||||
inliner.todo--;
|
||||
inliner.emit('jobs', (inliner.total - inliner.todo) + '/' + inliner.total);
|
||||
|
||||
// workaround for https://github.com/tmpvar/jsdom/issues/172
|
||||
// need to remove empty script tags as it casues jsdom to skip the env callback
|
||||
html = html.replace(/<\script(:? type=['"|].*?['"|])><\/script>/ig, '');
|
||||
|
||||
if (!html) {
|
||||
inliner.emit('end', '');
|
||||
callback && callback('');
|
||||
} else {
|
||||
// BIG ASS PROTECTIVE TRY/CATCH - mostly because of this: https://github.com/tmpvar/jsdom/issues/319
|
||||
try {
|
||||
|
||||
jsdom.env(html, [
|
||||
'http://code.jquery.com/jquery.min.js'
|
||||
], {
|
||||
url: url
|
||||
}, function(errors, window) {
|
||||
// remove jQuery that was included with jsdom
|
||||
window.$('script:last').remove();
|
||||
|
||||
var todo = { scripts: true, images: inliner.options.images, links: true, styles: true },
|
||||
assets = {
|
||||
scripts: window.$('script'),
|
||||
images: window.$('img').filter(function(){ return this.src.indexOf('data:') == -1; }),
|
||||
links: window.$('link[rel=stylesheet]'),
|
||||
styles: window.$('style')
|
||||
},
|
||||
breakdown = {},
|
||||
images = {};
|
||||
|
||||
inliner.total = 1;
|
||||
|
||||
for (var key in todo) {
|
||||
if (todo[key] === true && assets[key]) {
|
||||
breakdown[key] = assets[key].length;
|
||||
inliner.total += assets[key].length;
|
||||
inliner.todo += assets[key].length;
|
||||
} else {
|
||||
assets[key] = [];
|
||||
}
|
||||
}
|
||||
|
||||
inliner.emit('jobs', (inliner.total - inliner.todo) + '/' + inliner.total);
|
||||
|
||||
function finished() {
|
||||
var items = 0,
|
||||
html = '';
|
||||
for (var key in breakdown) {
|
||||
items += breakdown[key];
|
||||
}
|
||||
inliner.emit('jobs', (inliner.total - inliner.todo) + '/' + inliner.total);
|
||||
|
||||
if (items === 0) {
|
||||
// manually remove the comments
|
||||
var els = removeComments(window.document.documentElement);
|
||||
// collapse the white space
|
||||
if (inliner.options.collapseWhitespace) {
|
||||
// TODO put white space helper back in
|
||||
window.$('pre').html(function (i, html) {
|
||||
return html.replace(/\n/g, '~~nl~~'); //.replace(/\s/g, '~~s~~');
|
||||
});
|
||||
window.$('textarea').val(function (i, v) {
|
||||
return v.replace(/\n/g, '~~nl~~').replace(/\s/g, '~~s~~');
|
||||
});
|
||||
html = window.document.innerHTML;
|
||||
html = html.replace(/\s+/g, ' ').replace(/~~nl~~/g, '\n').replace(/~~s~~/g, ' ');
|
||||
} else {
|
||||
html = window.document.innerHTML;
|
||||
}
|
||||
|
||||
html = '<!DOCTYPE html>' + html;
|
||||
callback && callback(html);
|
||||
inliner.emit('end', html);
|
||||
} else if (items < 0) {
|
||||
console.log('something went wrong on finish');
|
||||
console.dir(breakdown);
|
||||
}
|
||||
}
|
||||
|
||||
todo.images && assets.images.each(function () {
|
||||
var img = this,
|
||||
resolvedURL = URL.resolve(url, img.src);
|
||||
inliner.get(resolvedURL, { encode: true }, function (dataurl) {
|
||||
if (dataurl) images[img.src] = dataurl;
|
||||
img.src = dataurl;
|
||||
breakdown.images--;
|
||||
inliner.todo--;
|
||||
finished();
|
||||
});
|
||||
});
|
||||
|
||||
todo.styles && assets.styles.each(function () {
|
||||
var style = this;
|
||||
inliner.getImportCSS(root, this.innerHTML, function (css, url) {
|
||||
inliner.getImagesFromCSS(url, css, function (css) {
|
||||
if (inliner.options.compressCSS) inliner.emit('progress', 'compress inline css');
|
||||
window.$(style).text(css);
|
||||
|
||||
breakdown.styles--;
|
||||
inliner.todo--;
|
||||
finished();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
todo.links && assets.links.each(function () {
|
||||
var link = this,
|
||||
linkURL = URL.resolve(url, link.href);
|
||||
|
||||
inliner.get(linkURL, function (css) {
|
||||
inliner.getImagesFromCSS(linkURL, css, function (css) {
|
||||
inliner.getImportCSS(linkURL, css, function (css) {
|
||||
if (inliner.options.compressCSS) inliner.emit('progress', 'compress ' + linkURL);
|
||||
breakdown.links--;
|
||||
inliner.todo--;
|
||||
|
||||
var style = '',
|
||||
media = link.getAttribute('media');
|
||||
|
||||
if (media) {
|
||||
style = '<style>@media ' + media + '{' + css + '}</style>';
|
||||
} else {
|
||||
style = '<style>' + css + '</style>';
|
||||
}
|
||||
|
||||
window.$(link).replaceWith(style);
|
||||
finished();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
function scriptsFinished() {
|
||||
inliner.emit('jobs', (inliner.total - inliner.todo) + '/' + inliner.total);
|
||||
if (breakdown.scripts == 0) {
|
||||
// now compress the source JavaScript
|
||||
assets.scripts.each(function () {
|
||||
if (this.innerHTML.trim().length == 0) {
|
||||
// this is an empty script, so throw it away
|
||||
inliner.todo--;
|
||||
inliner.emit('jobs', (inliner.total - inliner.todo) + '/' + inliner.total);
|
||||
return;
|
||||
}
|
||||
|
||||
var $script = window.$(this),
|
||||
src = $script.attr('src'),
|
||||
// note: not using .innerHTML as this coerses & => &
|
||||
orig_code = this.firstChild.nodeValue
|
||||
.replace(/<\/script>/gi, '<\\/script>'),
|
||||
final_code = '';
|
||||
|
||||
// only remove the src if we have a script body
|
||||
if (orig_code) {
|
||||
$script.removeAttr('src');
|
||||
}
|
||||
|
||||
// don't compress already minified code
|
||||
if(!(/\bmin\b/).test(src) && !(/google-analytics/).test(src)) {
|
||||
inliner.todo++;
|
||||
inliner.total++;
|
||||
inliner.emit('jobs', (inliner.total - inliner.todo) + '/' + inliner.total);
|
||||
try {
|
||||
var ast = jsp.parse(orig_code); // parse code and get the initial AST
|
||||
|
||||
ast = pro.ast_mangle(ast); // get a new AST with mangled names
|
||||
ast = pro.ast_squeeze(ast); // get an AST with compression optimizations
|
||||
final_code = pro.gen_code(ast);
|
||||
|
||||
// some protection against putting script tags in the body
|
||||
window.$(this).text(final_code).append('\n');
|
||||
|
||||
if (src) {
|
||||
inliner.emit('progress', 'compress ' + URL.resolve(root, src));
|
||||
} else {
|
||||
inliner.emit('progress', 'compress inline script');
|
||||
}
|
||||
} catch (e) {
|
||||
// console.error(orig_code.indexOf('script>script'));
|
||||
// window.$(this).html(jsmin('', orig_code, 2));
|
||||
console.error('exception on ', src);
|
||||
console.error('exception in ' + src + ': ' + e.message);
|
||||
console.error('>>>>>> ' + orig_code.split('\n')[e.line - 1]);
|
||||
}
|
||||
inliner.todo--;
|
||||
inliner.emit('jobs', (inliner.total - inliner.todo) + '/' + inliner.total);
|
||||
} else if (orig_code) {
|
||||
window.$(this).text(orig_code);
|
||||
// this.innerText = orig_code;
|
||||
}
|
||||
});
|
||||
finished();
|
||||
}
|
||||
}
|
||||
|
||||
// basically this is the jQuery instance we tacked on to the request,
|
||||
// but we're just being extra sure before we do zap it out
|
||||
todo.scripts && assets.scripts.each(function () {
|
||||
var $script = window.$(this),
|
||||
scriptURL = URL.resolve(url, (this.src||"").toString());
|
||||
|
||||
if (!this.src || scriptURL.indexOf('google-analytics.com') !== -1) { // ignore google
|
||||
breakdown.scripts--;
|
||||
inliner.todo--;
|
||||
scriptsFinished();
|
||||
} else if (this.src) {
|
||||
inliner.get(scriptURL, { not: 'text/html' }, function (data) {
|
||||
// catches an exception that was being thrown, but script escaping wasn't being caught
|
||||
if (data) $script.text(data.replace(/<\/script>/gi, '<\\/script>')); //.replace(/\/\/.*$\n/g, ''));
|
||||
// $script.before('<!-- ' + scriptURL + ' -->');
|
||||
breakdown.scripts--;
|
||||
inliner.todo--;
|
||||
scriptsFinished();
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// edge case - if there's no images, nor scripts, nor links - we call finished manually
|
||||
if (assets.links.length == 0 &&
|
||||
assets.styles.length == 0 &&
|
||||
assets.images.length == 0 &&
|
||||
assets.scripts.length == 0) {
|
||||
finished();
|
||||
}
|
||||
|
||||
/** Inliner jobs:
|
||||
* 1. get all inline images and base64 encode
|
||||
* 2. get all external style sheets and move to inline
|
||||
* 3. get all image references in CSS and base64 encode and replace urls
|
||||
* 4. get all external scripts and move to inline
|
||||
* 5. compress JavaScript
|
||||
* 6. compress CSS & support media queries
|
||||
* 7. compress HTML (/>\s+</g, '> <');
|
||||
*
|
||||
* FUTURE ITEMS:
|
||||
* - support for @import
|
||||
* - javascript validation - i.e. not throwing errors
|
||||
*/
|
||||
});
|
||||
|
||||
} catch (e) {
|
||||
inliner.emit('error', 'Fatal error parsing HTML - exiting');
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
util.inherits(Inliner, events.EventEmitter);
|
||||
|
||||
Inliner.version = Inliner.prototype.version = JSON.parse(require('fs').readFileSync(__dirname + '/package.json').toString()).version;
|
||||
|
||||
Inliner.prototype.get = function (url, options, callback) {
|
||||
// support no options being passed in
|
||||
if (typeof options == 'function') {
|
||||
callback = options;
|
||||
options = {};
|
||||
}
|
||||
|
||||
// if we've cached this request in the past, just return the cached content
|
||||
if (this.requestCache[url] !== undefined) {
|
||||
this.emit('progress', 'cached ' + url);
|
||||
return callback && callback(this.requestCache[url]);
|
||||
} else if (this.requestCachePending[url] !== undefined) {
|
||||
this.requestCachePending[url].push(callback);
|
||||
return true;
|
||||
} else {
|
||||
this.requestCachePending[url] = [callback];
|
||||
}
|
||||
|
||||
var inliner = this;
|
||||
|
||||
// TODO remove the sync
|
||||
if (fs.existsSync(url)) {
|
||||
// then we're dealing with a file
|
||||
fs.readFile(url, 'utf8', function (err, body) {
|
||||
inliner.requestCache[url] = body;
|
||||
inliner.requestCachePending[url].forEach(function (callback, i) {
|
||||
if (i == 0 && body) {
|
||||
inliner.emit('progress', (options.encode ? 'encode' : 'get') + ' ' + url);
|
||||
} else if (body) {
|
||||
inliner.emit('progress', 'cached ' + url);
|
||||
}
|
||||
callback && callback(body);
|
||||
});
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// otherwis continue and create a new web request
|
||||
var request = makeRequest(url),
|
||||
body = '';
|
||||
|
||||
// this tends to occur when we can't connect to the url - i.e. target is down
|
||||
// note that the main inliner app handles sending the error back to the client
|
||||
request.on('error', function (error) {
|
||||
console.error(error.message, url);
|
||||
callback && callback('');
|
||||
});
|
||||
|
||||
|
||||
request.on('response', function (res) {
|
||||
var gunzip;
|
||||
|
||||
// if we get a gzip header, then first we attempt to load the node-compress library
|
||||
// ...which annoyingly isn't supported anymore (maybe I should fork it...).
|
||||
// once loaded, we set up event listeners to handle data coming in and do a little
|
||||
// dance with the response object - which I'll explain... ==>
|
||||
if (res.headers['content-encoding'] == 'gzip') {
|
||||
console.error('loading gzip library');
|
||||
if (compress === null) {
|
||||
try {
|
||||
compress = require('./node-compress/lib/compress/');
|
||||
} catch (e) {
|
||||
console.error(url + ' sent gzipped header\nFailed to load node-compress - see http://github.com/remy/inliner for install directions. \nexiting');
|
||||
process.exit();
|
||||
}
|
||||
}
|
||||
gunzip = new compress.GunzipStream();
|
||||
|
||||
// the data event is triggered by writing to the gunzip object when the response
|
||||
// receives data (yeah, further down).
|
||||
// once all the data has finished (triggerd by the response end event), we undefine
|
||||
// the gunzip object and re-trigger the response end event. By this point, we've
|
||||
// decompressed the gzipped content, and it's human readable again.
|
||||
gunzip.on('data', function (chunk) {
|
||||
body += chunk;
|
||||
}).on('end', function () {
|
||||
gunzip = undefined;
|
||||
res.emit('end');
|
||||
});
|
||||
}
|
||||
|
||||
res.on('data', function (chunk) {
|
||||
// only process data if we have a 200 ok
|
||||
if (res.statusCode == 200) {
|
||||
if (gunzip) {
|
||||
gunzip.write(chunk);
|
||||
} else {
|
||||
body += chunk;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// required if we're going to base64 encode the content later on
|
||||
if (options.encode) {
|
||||
res.setEncoding('binary');
|
||||
}
|
||||
|
||||
res.on('end', function () {
|
||||
if (gunzip) {
|
||||
gunzip.end();
|
||||
return;
|
||||
}
|
||||
|
||||
if (res.statusCode !== 200) {
|
||||
inliner.emit('progress', 'get ' + res.statusCode + ' on ' + url);
|
||||
body = ''; // ?
|
||||
callback && callback(body);
|
||||
} else if (res.headers['location']) {
|
||||
return inliner.get(res.headers['location'], options, callback);
|
||||
} else {
|
||||
if (options && options.not) {
|
||||
if (res.headers['content-type'].indexOf(options.not) !== -1) {
|
||||
body = '';
|
||||
}
|
||||
}
|
||||
|
||||
if (options.encode && res.statusCode == 200) {
|
||||
body = 'data:' + res.headers['content-type'] + ';base64,' + new Buffer(body, 'binary').toString('base64');
|
||||
}
|
||||
|
||||
inliner.requestCache[url] = body;
|
||||
inliner.requestCachePending[url].forEach(function (callback, i) {
|
||||
if (i == 0 && body && res.statusCode == 200) {
|
||||
inliner.emit('progress', (options.encode ? 'encode' : 'get') + ' ' + url);
|
||||
} else if (body) {
|
||||
inliner.emit('progress', 'cached ' + url);
|
||||
}
|
||||
callback && callback(body);
|
||||
});
|
||||
}
|
||||
});
|
||||
}).end();
|
||||
};
|
||||
|
||||
Inliner.prototype.getImagesFromCSS = function (rooturl, rawCSS, callback) {
|
||||
if (this.options.images === false) {
|
||||
callback && callback(rawCSS);
|
||||
return;
|
||||
}
|
||||
|
||||
var inliner = this,
|
||||
images = {},
|
||||
urlMatch = /url\((?:['"]*)(?!['"]*data:)(.*?)(?:['"]*)\)/g,
|
||||
singleURLMatch = /url\(\s*(?:['"]*)(?!['"]*data:)(.*?)(?:['"]*)\s*\)/,
|
||||
matches = rawCSS.match(urlMatch),
|
||||
imageCount = matches === null ? 0 : matches.length; // TODO check!
|
||||
|
||||
inliner.total += imageCount;
|
||||
inliner.todo += imageCount;
|
||||
|
||||
function checkFinished() {
|
||||
inliner.emit('jobs', (inliner.total - inliner.todo) + '/' + inliner.total);
|
||||
if (imageCount < 0) {
|
||||
console.log('something went wrong :-S');
|
||||
} else if (imageCount == 0) {
|
||||
callback(rawCSS.replace(urlMatch, function (m, url) {
|
||||
return 'url(' + images[url] + ')';
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
if (imageCount) {
|
||||
matches.forEach(function (url) {
|
||||
url = url.match(singleURLMatch)[1];
|
||||
var resolvedURL = URL.resolve(rooturl, url);
|
||||
if (images[url] === undefined) {
|
||||
inliner.get(resolvedURL, { encode: true }, function (dataurl) {
|
||||
imageCount--;
|
||||
inliner.todo--;
|
||||
if (images[url] === undefined) images[url] = dataurl;
|
||||
|
||||
checkFinished();
|
||||
});
|
||||
} else {
|
||||
imageCount--;
|
||||
inliner.todo--;
|
||||
checkFinished();
|
||||
}
|
||||
});
|
||||
} else {
|
||||
callback(rawCSS);
|
||||
}
|
||||
};
|
||||
|
||||
Inliner.prototype.getImportCSS = function (rooturl, css, callback) {
|
||||
// if (typeof css == 'function') {
|
||||
// callback = css;
|
||||
// rooturl = '';
|
||||
// }
|
||||
|
||||
var position = css.indexOf('@import'),
|
||||
inliner = this;
|
||||
|
||||
if (position !== -1) {
|
||||
var match = css.match(/@import\s*(.*)/);
|
||||
|
||||
if (match !== null && match.length) {
|
||||
var url = match[1].replace(/url/, '').replace(/['}"]/g, '').replace(/;/, '').trim().split(' '); // clean up
|
||||
// if url has a length > 1, then we have media types to target
|
||||
var resolvedURL = URL.resolve(rooturl, url[0]);
|
||||
inliner.get(resolvedURL, function (importedCSS) {
|
||||
inliner.emit('progress', 'import ' + resolvedURL);
|
||||
if (url.length > 1) {
|
||||
url.shift();
|
||||
importedCSS = '@media ' + url.join(' ') + '{' + importedCSS + '}';
|
||||
}
|
||||
|
||||
css = css.replace(match[0], importedCSS);
|
||||
inliner.getImportCSS(rooturl, css, callback);
|
||||
});
|
||||
}
|
||||
} else {
|
||||
if (inliner.options.compressCSS) css = compressCSS(css);
|
||||
callback(css, rooturl);
|
||||
}
|
||||
};
|
||||
|
||||
Inliner.defaults = function () { return { compressCSS: true, collapseWhitespace: true, images: true }; };
|
||||
|
||||
var makeRequest = Inliner.makeRequest = function (url, extraOptions) {
|
||||
var oURL = URL.parse(url),
|
||||
options = {
|
||||
host: oURL.hostname,
|
||||
port: oURL.port === undefined ? (oURL.protocol+'').indexOf('https') === 0 ? 443 : 80 : oURL.port,
|
||||
path: (oURL.pathname || '/') + (oURL.search || ''), // note 0.5.0pre doesn't fill pathname if missing
|
||||
method: 'GET'
|
||||
};
|
||||
|
||||
for (var key in extraOptions) {
|
||||
options[key] = extraOptions[key];
|
||||
}
|
||||
|
||||
return http[oURL.protocol.slice(0, -1) || 'http'].request(options);
|
||||
};
|
||||
|
||||
module.exports = Inliner;
|
||||
|
||||
if (!module.parent) {
|
||||
// if this module isn't being included in a larger app, defer to the
|
||||
// bin/inliner for the help options
|
||||
require('./bin/inliner');
|
||||
}
|
||||
3
lib/errors.js
Normal file
3
lib/errors.js
Normal file
@ -0,0 +1,3 @@
|
||||
module.exports = {
|
||||
ENOTFOUND: 'The URL could not be loaded, possible typo?',
|
||||
};
|
||||
20
lib/find-assets.js
Normal file
20
lib/find-assets.js
Normal file
@ -0,0 +1,20 @@
|
||||
module.exports = findAssets;
|
||||
|
||||
var cheerio = require('cheerio');
|
||||
|
||||
function findAssets(html) {
|
||||
var $ = cheerio.load(html);
|
||||
|
||||
var images = $('img');
|
||||
var links = $('link[rel=stylesheet]');
|
||||
var styles = $('style');
|
||||
var scripts = $('script');
|
||||
|
||||
return {
|
||||
$: $,
|
||||
images: images,
|
||||
links: links,
|
||||
styles: styles,
|
||||
js: scripts,
|
||||
};
|
||||
}
|
||||
34
lib/get.js
Normal file
34
lib/get.js
Normal file
@ -0,0 +1,34 @@
|
||||
var request = require('request');
|
||||
var assign = require('lodash.assign');
|
||||
var debug = require('debug')('inliner');
|
||||
|
||||
var cache = {};
|
||||
|
||||
module.exports = function get(url, options) {
|
||||
debug('request %s', url);
|
||||
|
||||
var settings = assign({}, options, {
|
||||
followRedirect: true,
|
||||
});
|
||||
|
||||
if (cache[url]) {
|
||||
return Promise.resolve(cache[url]);
|
||||
}
|
||||
|
||||
return new Promise(function promise(resolve, reject) {
|
||||
request(url, settings, function response(error, res, body) {
|
||||
if (error) {
|
||||
debug('request failed: %s', error.message);
|
||||
return reject(error);
|
||||
}
|
||||
|
||||
debug('response: %s %s', res.statusCode, url);
|
||||
if (!error && res.statusCode === 200) {
|
||||
cache[url] = { headers: res.headers, body: body };
|
||||
resolve(cache[url]);
|
||||
} else {
|
||||
return reject(new Error(res.statusCode));
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
15
lib/image.js
Normal file
15
lib/image.js
Normal file
@ -0,0 +1,15 @@
|
||||
module.exports = image;
|
||||
|
||||
var get = require('./get');
|
||||
var debug = require('debug')('inliner');
|
||||
|
||||
function image(url) {
|
||||
return get(url, { encoding: 'binary' }).then(function then(res) {
|
||||
debug('image loaded: %s', url);
|
||||
var buffer = new Buffer(res.body, 'binary').toString('base64');
|
||||
return 'data:' + res.headers['content-type'] + ';base64,' + buffer;
|
||||
}).catch(function errorHandle(error) {
|
||||
debug('image %s failed to load', url, error);
|
||||
throw error;
|
||||
});
|
||||
}
|
||||
170
lib/index.js
Normal file
170
lib/index.js
Normal file
@ -0,0 +1,170 @@
|
||||
module.exports = Inliner;
|
||||
|
||||
var debug = require('debug')('inliner');
|
||||
var events = require('events');
|
||||
var path = require('path');
|
||||
var util = require('util');
|
||||
var fs = require('then-fs');
|
||||
var assign = require('lodash.assign');
|
||||
var forEach = require('lodash.foreach');
|
||||
var Promise = require('es6-promise').Promise; // jshint ignore:line
|
||||
var get = require('./get');
|
||||
var findAssets = require('./find-assets');
|
||||
var getImage = require('./image');
|
||||
var resolveURL = require('url').resolve;
|
||||
var resolveFile = require('path').resolve;
|
||||
|
||||
function Inliner(url, options, callback) {
|
||||
var inliner = this;
|
||||
this.url = url;
|
||||
this.options = assign({}, options, Inliner.defaults());
|
||||
this.total = 0;
|
||||
this.todo = 0;
|
||||
this.breakdown = {
|
||||
html: 0,
|
||||
js: 0,
|
||||
links: 0,
|
||||
styles: 0,
|
||||
images: 0,
|
||||
};
|
||||
// allows us to use in a promise chain without calling
|
||||
this.jobs = {
|
||||
add: this.addJob.bind(this),
|
||||
done: {
|
||||
html: this.completeJob.bind(this, 'html'),
|
||||
js: this.completeJob.bind(this, 'js'),
|
||||
images: this.completeJob.bind(this, 'images'),
|
||||
links: this.completeJob.bind(this, 'links'),
|
||||
styles: this.completeJob.bind(this, 'styles'),
|
||||
},
|
||||
};
|
||||
|
||||
var resolve = resolveURL;
|
||||
|
||||
events.EventEmitter.call(this);
|
||||
|
||||
// this allows the user code to get the object back before
|
||||
// it starts firing events
|
||||
setImmediate(function immediate() {
|
||||
var promise = fs.exists(url).then(function exists(isFile) {
|
||||
if (!isFile) {
|
||||
throw new Error();
|
||||
}
|
||||
|
||||
resolve = resolveFile;
|
||||
|
||||
debug('loading file:s %s', url);
|
||||
|
||||
return fs.readFile(url, 'utf8');
|
||||
}).catch(function isUrl() {
|
||||
// check for protocol on URL
|
||||
if (url.indexOf('http') !== 0) {
|
||||
url = 'http://' + url;
|
||||
}
|
||||
|
||||
debug('loading url: %s', url);
|
||||
|
||||
return get(url);
|
||||
}).then(inliner.jobs.add('html'));
|
||||
|
||||
promise.then(function processHTML(res) {
|
||||
inliner.jobs.done.html();
|
||||
debug('then');
|
||||
|
||||
var todo = findAssets(res.body);
|
||||
var $ = todo.$;
|
||||
delete todo.$;
|
||||
|
||||
forEach(todo, function forEach(todo, key) {
|
||||
if (key === 'images' && !inliner.options.images) {
|
||||
// skip images if the user doesn't want them
|
||||
delete todo.images;
|
||||
debug('skipping images');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!todo.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
debug('assets %s: %s', key, todo.length);
|
||||
inliner.jobs.add(key, todo.length);
|
||||
});
|
||||
|
||||
var promises = [];
|
||||
|
||||
if (todo.images) {
|
||||
var imagePromises = todo.images.map(function images(i, image) {
|
||||
var url = resolve(inliner.url, $(image).attr('src'));
|
||||
return getImage(url).then(function then(dataURL) {
|
||||
$(image).attr('src', dataURL);
|
||||
}).then(inliner.jobs.done.images);
|
||||
}).get();
|
||||
|
||||
promises = promises.concat(imagePromises);
|
||||
}
|
||||
|
||||
return Promise.all(promises).then(function then() {
|
||||
return $.html();
|
||||
});
|
||||
|
||||
})
|
||||
.then(function then(html) {
|
||||
callback(null, html);
|
||||
})
|
||||
.catch(function errHandler(error) {
|
||||
debug('fail', error);
|
||||
callback(error);
|
||||
inliner.emit('error', error);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
util.inherits(Inliner, events.EventEmitter);
|
||||
|
||||
Inliner.prototype.updateTodo = function updateTodo() {
|
||||
this.todo = this.breakdown.html +
|
||||
this.breakdown.js +
|
||||
this.breakdown.links +
|
||||
this.breakdown.styles +
|
||||
this.breakdown.images +
|
||||
0;
|
||||
|
||||
this.emit('jobs', (this.total - this.todo) + '/' + this.total);
|
||||
};
|
||||
|
||||
Inliner.prototype.addJob = function addJob(type) {
|
||||
var n = typeof arguments[1] === 'number' ? arguments[1] : 1;
|
||||
this.breakdown[type] += n;
|
||||
this.total += n;
|
||||
this.updateTodo();
|
||||
|
||||
// this allows me to include addJob as part of a promise chain
|
||||
return arguments[1];
|
||||
};
|
||||
|
||||
Inliner.prototype.completeJob = function completeJob(type) {
|
||||
this.breakdown[type]--;
|
||||
this.updateTodo();
|
||||
|
||||
// this allows me to include addJob as part of a promise chain
|
||||
return arguments[1];
|
||||
};
|
||||
|
||||
// start of static methods
|
||||
|
||||
// make the version available
|
||||
Inliner.version =
|
||||
Inliner.prototype.version =
|
||||
require(path.resolve(__dirname, '..', 'package.json')).version;
|
||||
|
||||
Inliner.errors = require('./errors');
|
||||
|
||||
Inliner.defaults = function() {
|
||||
return {
|
||||
images: true,
|
||||
compressCSS: true,
|
||||
collapseWhitespace: true,
|
||||
images: true,
|
||||
};
|
||||
};
|
||||
24
package.json
24
package.json
@ -1,9 +1,9 @@
|
||||
{
|
||||
"name": "inliner",
|
||||
"version": "0.1.16",
|
||||
"version": "1.0.0",
|
||||
"description": "Utility to inline images, CSS and JavaScript for a web page - useful for mobile sites",
|
||||
"homepage": "http://github.com/remy/inliner",
|
||||
"main": "inliner",
|
||||
"main": "lib/index.js",
|
||||
"keywords": [
|
||||
"mobile",
|
||||
"inline",
|
||||
@ -11,20 +11,24 @@
|
||||
"build",
|
||||
"minify"
|
||||
],
|
||||
"author": {
|
||||
"name": "Remy Sharp",
|
||||
"web": "http://github.com/remy"
|
||||
},
|
||||
"author": "Remy Sharp",
|
||||
"dependencies": {
|
||||
"jsdom": "0.6.5",
|
||||
"uglify-js": "1.2.2",
|
||||
"commander": "0.5.1"
|
||||
"cheerio": "^0.19.0",
|
||||
"debug": "^2.2.0",
|
||||
"es6-promise": "^2.3.0",
|
||||
"lodash.assign": "^3.2.0",
|
||||
"lodash.foreach": "^3.0.3",
|
||||
"minimist": "^1.1.2",
|
||||
"request": "^2.60.0",
|
||||
"then-fs": "^2.0.0",
|
||||
"uglify-js": "^2.4.24"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git://github.com/remy/inliner.git"
|
||||
},
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"inliner": "bin/inliner"
|
||||
"inliner": "cli/index.js"
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user