UglifyJsPlugin: extract comments to separate file

SebastianS90 · SebastianS90 · commit 71933e979e51 · 2017-02-10T11:59:56.000+01:00
License comments use up a lot of space, especially when using many small
libraries with large license blocks. With this addition, you can extract
all license comments to a separate file and remove them from the bundle
files. A small banner points to the file containing all license
information such that the user can find it if needed.

We add a new option extractComments to the UglifyJsPlugin.
It can be omitted, then the behavior does not change, or it can be:
- true: All comments that normally would be preserved by the comments
    option will be moved to a separate file. If the original file is
    named foo.js, then the comments will be stored to foo.js.LICENSE
- regular expression (given as RegExp or string) or a function
  (astNode, comment) -&gt; boolean: All comments that match the given
    expression (resp. are evaluated to true by the function) will be
    extracted to the separate file. The comments option specifies
    whether the comment will be preserved, i.e. it is possible to
    preserve some comments (e.g. annotations) while extracting others or
    even preserving comments that have been extracted.
- an object consisting of the following keys, all optional:
  - condition: regular expression or function (see previous point)
  - file: The file where the extracted comments will be stored. Can be
      either a string (filename) or function (string) -&gt; string which
      will be given the original filename. Default is to append the
      suffix .LICENSE to the original filename.
  - banner: The banner text that points to the extracted file and will
      be added on top of the original file. will be added to the
      original file. Can be false (no banner), a string, or a function
      (string) -&gt; string that will be called with the filename where
      extracted comments have been stored. Will be wrapped into comment.
      Default: /*! For license information please see foo.js.LICENSE */
diff --git a/lib/optimize/UglifyJsPlugin.js b/lib/optimize/UglifyJsPlugin.js
@@ -7,6 +7,7 @@
 const SourceMapConsumer = require("source-map").SourceMapConsumer;
 const SourceMapSource = require("webpack-sources").SourceMapSource;
 const RawSource = require("webpack-sources").RawSource;
+const ConcatSource = require("webpack-sources").ConcatSource;
 const RequestShortener = require("../RequestShortener");
 const ModuleFilenameHelpers = require("../ModuleFilenameHelpers");
 const uglify = require("uglify-js");
@@ -102,6 +103,54 @@ class UglifyJsPlugin {
 						for(let k in options.output) {
 							output[k] = options.output[k];
 						}
+						const extractedComments = [];
+						if(options.extractComments) {
+							const condition = {};
+							if(typeof options.extractComments === "string" || options.extractComments instanceof RegExp) {
+								// extractComments specifies the extract condition and output.comments specifies the preserve condition
+								condition.preserve = output.comments;
+								condition.extract = options.extractComments;
+							} else if(Object.prototype.hasOwnProperty.call(options.extractComments, "condition")) {
+								// Extract condition is given in extractComments.condition
+								condition.preserve = output.comments;
+								condition.extract = options.extractComments.condition;
+							} else {
+								// No extract condition is given. Extract comments that match output.comments instead of preserving them
+								condition.preserve = false;
+								condition.extract = output.comments;
+							}
+
+							// Ensure that both conditions are functions
+							["preserve", "extract"].forEach(key => {
+								switch(typeof condition[key]) {
+									case "boolean":
+										var b = condition[key];
+										condition[key] = () => b;
+									case "function": // eslint-disable-line no-fallthrough
+										break;
+									case "string":
+										if(condition[key] === "all") {
+											condition[key] = () => true;
+											break;
+										}
+										condition[key] = new RegExp(condition[key]);
+									default: // eslint-disable-line no-fallthrough
+										var regex = condition[key];
+										condition[key] = (astNode, comment) => regex.test(comment.value);
+								}
+							});
+
+							// Redefine the comments function to extract and preserve
+							// comments according to the two conditions
+							output.comments = (astNode, comment) => {
+								if(condition.extract(astNode, comment)) {
+									extractedComments.push(
+										comment.type === "comment2" ? "/*" + comment.value + "*/" : "//" + comment.value
+									);
+								}
+								return condition.preserve(astNode, comment);
+							};
+						}
 						let map;
 						if(options.sourceMap) {
 							map = uglify.SourceMap({ // eslint-disable-line new-cap
@@ -117,6 +166,41 @@ class UglifyJsPlugin {
 						asset.__UglifyJsPlugin = compilation.assets[file] = (map ?
 							new SourceMapSource(stringifiedStream, file, JSON.parse(map), input, inputSourceMap) :
 							new RawSource(stringifiedStream));
+						if(extractedComments.length > 0) {
+							let commentsFile = options.extractComments.file || file + ".LICENSE";
+							if(typeof commentsFile === "function") {
+								commentsFile = commentsFile(file);
+							}
+
+							// Write extracted comments to commentsFile
+							const commentsSource = new RawSource(extractedComments.join("\n\n") + "\n");
+							if(commentsFile in compilation.assets) {
+								// commentsFile already exists, append new comments...
+								if(compilation.assets[commentsFile] instanceof ConcatSource) {
+									compilation.assets[commentsFile].add("\n");
+									compilation.assets[commentsFile].add(commentsSource);
+								} else {
+									compilation.assets[commentsFile] = new ConcatSource(
+										compilation.assets[commentsFile], "\n", commentsSource
+									);
+								}
+							} else {
+								compilation.assets[commentsFile] = commentsSource;
+							}
+
+							// Add a banner to the original file
+							if(options.extractComments.banner !== false) {
+								let banner = options.extractComments.banner || "For license information please see " + commentsFile;
+								if(typeof banner === "function") {
+									banner = banner(commentsFile);
+								}
+								if(banner) {
+									asset.__UglifyJsPlugin = compilation.assets[file] = new ConcatSource(
+										"/*! " + banner + " */\n", compilation.assets[file]
+									);
+								}
+							}
+						}
 						if(warnings.length > 0) {
 							compilation.warnings.push(new Error(file + " from UglifyJs\n" + warnings.join("\n")));
 						}
diff --git a/test/UglifyJsPlugin.test.js b/test/UglifyJsPlugin.test.js
@@ -224,7 +224,16 @@ describe("UglifyJsPlugin", function() {
 				},
 				mangle: false,
 				beautify: true,
-				comments: false
+				comments: false,
+				extractComments: {
+					condition: 'should be extracted',
+					file: function(file) {
+						return file.replace(/(\.\w+)$/, '.license$1');
+					},
+					banner: function(licenseFile) {
+						return 'License information can be found in ' + licenseFile;
+					}
+				}
 			});
 			plugin.apply(compilerEnv);
 			eventBindings = pluginEnvironment.getEventBindings();
@@ -305,6 +314,19 @@ describe("UglifyJsPlugin", function() {
 								};
 							},
 						},
+						"test4.js": {
+							source: function() {
+								return "/*! this comment should be extracted */ function foo(longVariableName) { /* this will not be extracted */ longVariableName = 1; } // another comment that should be extracted to a separate file\n function foo2(bar) { return bar; }";
+							},
+							map: function() {
+								return {
+									version: 3,
+									sources: ["test.js"],
+									names: ["foo", "longVariableName"],
+									mappings: "AAAA,QAASA,KAAIC,kBACTA,iBAAmB"
+								};
+							}
+						},
 					};
 					compilation.errors = [];
 					compilation.warnings = [];
@@ -524,6 +546,120 @@ describe("UglifyJsPlugin", function() {
 							});
 						});
 					});
+
+					it("extracts license information to separate file", function() {
+						compilationEventBinding.handler([{
+							files: ["test4.js"]
+						}], function() {
+							compilation.errors.length.should.be.exactly(0);
+							compilation.assets["test4.license.js"]._value.should.containEql("/*! this comment should be extracted */");
+							compilation.assets["test4.license.js"]._value.should.containEql("// another comment that should be extracted to a separate file");
+							compilation.assets["test4.license.js"]._value.should.not.containEql("/* this will not be extracted */");
+						});
+					});
+				});
+			});
+		});
+	});
+
+	describe("when applied with extract option set to a single file", function() {
+		let eventBindings;
+		let eventBinding;
+
+		beforeEach(function() {
+			const pluginEnvironment = new PluginEnvironment();
+			const compilerEnv = pluginEnvironment.getEnvironmentStub();
+			compilerEnv.context = "";
+
+			const plugin = new UglifyJsPlugin({
+				comments: "all",
+				extractComments: {
+					condition: /.*/,
+					file: "extracted-comments.js"
+				}
+			});
+			plugin.apply(compilerEnv);
+			eventBindings = pluginEnvironment.getEventBindings();
+		});
+
+		it("binds one event handler", function() {
+			eventBindings.length.should.be.exactly(1);
+		});
+
+		describe("compilation handler", function() {
+			beforeEach(function() {
+				eventBinding = eventBindings[0];
+			});
+
+			it("binds to compilation event", function() {
+				eventBinding.name.should.be.exactly("compilation");
+			});
+
+			describe("when called", function() {
+				let chunkPluginEnvironment;
+				let compilationEventBindings;
+				let compilationEventBinding;
+				let compilation;
+
+				beforeEach(function() {
+					chunkPluginEnvironment = new PluginEnvironment();
+					compilation = chunkPluginEnvironment.getEnvironmentStub();
+					compilation.assets = {
+						"test.js": {
+							source: function() {
+								return "/* This is a comment from test.js */ function foo(bar) { return bar; }";
+							}
+						},
+						"test2.js": {
+							source: function() {
+								return "// This is a comment from test2.js\nfunction foo2(bar) { return bar; }";
+							}
+						},
+						"test3.js": {
+							source: function() {
+								return "/* This is a comment from test3.js */ function foo3(bar) { return bar; }\n// This is another comment from test3.js\nfunction foobar3(baz) { return baz; }";
+							}
+						},
+					};
+					compilation.errors = [];
+					compilation.warnings = [];
+
+					eventBinding.handler(compilation);
+					compilationEventBindings = chunkPluginEnvironment.getEventBindings();
+				});
+
+				it("binds one event handler", function() {
+					compilationEventBindings.length.should.be.exactly(1);
+				});
+
+				describe("optimize-chunk-assets handler", function() {
+					beforeEach(function() {
+						compilationEventBinding = compilationEventBindings[0];
+					});
+
+					it("preserves comments", function() {
+						compilationEventBinding.handler([{
+							files: ["test.js", "test2.js", "test3.js"]
+						}], function() {
+							compilation.assets["test.js"].source().should.containEql("/*");
+							compilation.assets["test2.js"].source().should.containEql("//");
+							compilation.assets["test3.js"].source().should.containEql("/*");
+							compilation.assets["test3.js"].source().should.containEql("//");
+						});
+					});
+
+					it("extracts comments to specified file", function() {
+						compilationEventBinding.handler([{
+							files: ["test.js", "test2.js", "test3.js"]
+						}], function() {
+							compilation.errors.length.should.be.exactly(0);
+							compilation.assets["extracted-comments.js"].source().should.containEql("/* This is a comment from test.js */");
+							compilation.assets["extracted-comments.js"].source().should.containEql("// This is a comment from test2.js");
+							compilation.assets["extracted-comments.js"].source().should.containEql("/* This is a comment from test3.js */");
+							compilation.assets["extracted-comments.js"].source().should.containEql("// This is another comment from test3.js");
+							compilation.assets["extracted-comments.js"].source().should.not.containEql("function");
+						});
+					});
 				});
 			});
 		});
diff --git a/test/configCases/plugins/uglifyjs-plugin/extract.js b/test/configCases/plugins/uglifyjs-plugin/extract.js
@@ -0,0 +1,15 @@
+/** @preserve comment should be extracted extract-test.1 */
+
+var foo = {};
+
+// comment should be stripped extract-test.2
+
+/*!
+ * comment should be extracted extract-test.3
+ */
+
+/**
+ * comment should be stripped extract-test.4
+ */
+
+module.exports = foo;
diff --git a/test/configCases/plugins/uglifyjs-plugin/index.js b/test/configCases/plugins/uglifyjs-plugin/index.js
@@ -23,5 +23,25 @@ it("should pass mangle options", function() {
 	source.should.containEql("function r(n){return function(n){try{t()}catch(t){n(t)}}}");
 });
 
+it("should extract comments to separate file", function() {
+	var fs = require("fs"),
+		path = require("path");
+	var source = fs.readFileSync(path.join(__dirname, "extract.js.LICENSE"), "utf-8");
+	source.should.containEql("comment should be extracted extract-test.1");
+	source.should.not.containEql("comment should be stripped extract-test.2");
+	source.should.containEql("comment should be extracted extract-test.3");
+	source.should.not.containEql("comment should be stripped extract-test.4");
+});
+
+it("should remove extracted comments and insert a banner", function() {
+	var fs = require("fs"),
+		path = require("path");
+	var source = fs.readFileSync(path.join(__dirname, "extract.js"), "utf-8");
+	source.should.not.containEql("comment should be extracted extract-test.1");
+	source.should.not.containEql("comment should be stripped extract-test.2");
+	source.should.not.containEql("comment should be extracted extract-test.3");
+	source.should.not.containEql("comment should be stripped extract-test.4");
+	source.should.containEql("/*! For license information please see extract.js.LICENSE */");
+});
 
 require.include("./test.js");
diff --git a/test/configCases/plugins/uglifyjs-plugin/webpack.config.js b/test/configCases/plugins/uglifyjs-plugin/webpack.config.js
@@ -7,18 +7,26 @@ module.exports = {
 	entry: {
 		bundle0: ["./index.js"],
 		vendors: ["./vendors.js"],
-		ie8: ["./ie8.js"]
+		ie8: ["./ie8.js"],
+		extract: ["./extract.js"]
 	},
 	output: {
 		filename: "[name].js"
 	},
 	plugins: [
 		new webpack.optimize.UglifyJsPlugin({
 			comments: false,
-			exclude: ["vendors.js"],
+			exclude: ["vendors.js", "extract.js"],
 			mangle: {
 				screw_ie8: false
 			}
-		})
+		}),
+		new webpack.optimize.UglifyJsPlugin({
+			extractComments: true,
+			include: ["extract.js"],
+			mangle: {
+				screw_ie8: false
+			}
+		}),
 	]
 };
