Use StringDecoder for Buffers in WritableStream

ajafff · fb55 · commit 2aad069ce479 · 2016-04-01T21:34:17.000+03:00
https://stackoverflow.com/questions/12121775/convert-streamed-buffers-to-utf8-string
diff --git a/lib/WritableStream.js b/lib/WritableStream.js
@@ -1,21 +1,25 @@
 module.exports = Stream;
 
 var Parser = require("./Parser.js"),
-    WritableStream = require("stream").Writable || require("readable-stream").Writable;
+    WritableStream = require("stream").Writable || require("readable-stream").Writable,
+    StringDecoder = require("string_decoder").StringDecoder,
+    Buffer = require("buffer").Buffer;
 
 function Stream(cbs, options){
 	var parser = this._parser = new Parser(cbs, options);
+	var decoder = this._decoder = new StringDecoder();
 
 	WritableStream.call(this, {decodeStrings: false});
 
 	this.once("finish", function(){
-		parser.end();
+		parser.end(decoder.end());
 	});
 }
 
 require("inherits")(Stream, WritableStream);
 
 WritableStream.prototype._write = function(chunk, encoding, cb){
+	if(chunk instanceof Buffer) chunk = this._decoder.write(chunk);
 	this._parser.write(chunk);
 	cb();
 };
diff --git a/test/unicode.js b/test/unicode.js
@@ -0,0 +1,21 @@
+var htmlparser2 = require(".."),
+    assert = require("assert");
+
+describe("WritableStream", function(){
+
+	it("should decode fragmented unicode characters", function(){
+		var processed = false;
+		var stream = new htmlparser2.WritableStream({
+			ontext: function(text){
+				assert.equal(text, "€");
+				processed = true;
+			}
+		});
+
+		stream.write(new Buffer([0xE2, 0x82]));
+		stream.write(new Buffer([0xAC]));
+		stream.end();
+
+		assert(processed);
+	});
+});
