From f1890735857cbf5b3fc54142afccc90c7a57c832 Mon Sep 17 00:00:00 2001 From: Timo Uhlmann Date: Thu, 4 Sep 2014 08:37:10 +0200 Subject: [PATCH 1/5] Specs for irc.util.fromSocketData --- package/bin/irc/irc_util.js | 26 +++++++++++++++++++- test/irc_utils_test.js | 47 +++++++++++++++++++++++++++++++++++++ test/test_runner.html | 1 + 3 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 test/irc_utils_test.js diff --git a/package/bin/irc/irc_util.js b/package/bin/irc/irc_util.js index fd13e2ec..e2091511 100644 --- a/package/bin/irc/irc_util.js +++ b/package/bin/irc/irc_util.js @@ -175,7 +175,31 @@ return f.readAsArrayBuffer(blob); }; + var dumpBuffer = function(buffer) { + console.groupCollapsed('<= Buffer [' + buffer.byteLength + ']'); + try { + var view, arr, hexes; + view = new Uint8Array(buffer); + arr = view.subarray(); + arr = Array.prototype.slice.call(arr); + hexes = arr.map(function(n) { + return Number(n).toString(16).toUpperCase(); + }); + // var offsets = new Array(16); + // for(var i=0; i<16; i++) { + // offsets[i] = Number(i).toString(16).toUpperCase(); + // } + // console.table([offsets, hexes]); + console.log(arr); + console.log(hexes.join(' ')); + } catch(e) { + console.err(e); + } + console.groupEnd(); + }; + arrayBuffer2String = function(buf, callback) { + dumpBuffer(buf); var blob, f; exports.arrayBufferConversionCount++; blob = createBlob(buf); @@ -184,7 +208,7 @@ exports.arrayBufferConversionCount--; return callback(e.target.result); }; - return f.readAsText(blob); + return f.readAsText(blob, 'ISO-8859-1'); }; }).call(this); diff --git a/test/irc_utils_test.js b/test/irc_utils_test.js new file mode 100644 index 00000000..dea61e15 --- /dev/null +++ b/test/irc_utils_test.js @@ -0,0 +1,47 @@ +var utf8 = '✓'; +(function() { + "use strict"; + + describe("IRC Utils provides the following functions:", function() { + + var waitsForArrayBufferConversion; + waitsForArrayBufferConversion = function() { + return waitsFor((function() { + return !window.irc.util.isConvertingArrayBuffers(); + }), 'wait for array buffer conversion', 500); + }; + + describe("fromSocketData", function() { + var fromSocketData = irc.util.fromSocketData; + var ab, cb; + beforeEach(function() { + cb = jasmine.createSpy('cb'); + }); + + it("calls the callback", function() { + ab = irc.util.arrayToArrayBuffer([65]); + fromSocketData(ab, cb); + waitsForArrayBufferConversion(); + return runs(function() { + expect(cb).toHaveBeenCalledWith('A'); + }); + }); + + describe("handles encoding", function() { + + [ + 'ISO 8859-1' + ].forEach(function(encoding) { + it(encoding, function() { + ab = irc.util.arrayToArrayBuffer([116, 115, 99, 104, 246]); + fromSocketData(ab, cb); + waitsForArrayBufferConversion(); + return runs(function() { + expect(cb).toHaveBeenCalledWith('tschö✓'); + }); + }); + }); + }); + }); + }); +})(); diff --git a/test/test_runner.html b/test/test_runner.html index 6d37a429..d6c4d7ba 100644 --- a/test/test_runner.html +++ b/test/test_runner.html @@ -91,6 +91,7 @@ + From b569dd4cf17a10df3aa72e9a43faafc8b680e8dc Mon Sep 17 00:00:00 2001 From: Timo Uhlmann Date: Thu, 4 Sep 2014 14:56:35 +0200 Subject: [PATCH 2/5] Using html5 doctype and setting charset to utf-8 in spec runner to make Chrome parse test js as utf-8 --- test/test_runner.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_runner.html b/test/test_runner.html index d6c4d7ba..0fec6a6a 100644 --- a/test/test_runner.html +++ b/test/test_runner.html @@ -1,7 +1,7 @@ - + + Jasmine Spec Runner From f0fcfc1d165c1a69477fbd20ffe38f4085901cc1 Mon Sep 17 00:00:00 2001 From: Timo Uhlmann Date: Thu, 4 Sep 2014 14:58:01 +0200 Subject: [PATCH 3/5] Falling back to ISO-8859-1 if parsing with UTF-8 led to replacement characters --- package/bin/irc/irc_util.js | 15 +++++++++++---- test/irc_utils_test.js | 23 ++++++++--------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/package/bin/irc/irc_util.js b/package/bin/irc/irc_util.js index e2091511..30f652ca 100644 --- a/package/bin/irc/irc_util.js +++ b/package/bin/irc/irc_util.js @@ -200,15 +200,22 @@ arrayBuffer2String = function(buf, callback) { dumpBuffer(buf); - var blob, f; + var blob, f, hadReplacements; exports.arrayBufferConversionCount++; blob = createBlob(buf); f = new FileReader(); + hadReplacements = false; f.onload = function(e) { - exports.arrayBufferConversionCount--; - return callback(e.target.result); + if(e.target.result.match(/\ufffd/) && !hadReplacements) { + console.log('Retrying with ISO-8859-1', e.target.result); + hadReplacements = true; + return f.readAsText(blob, 'ISO-8859-1'); + } else { + exports.arrayBufferConversionCount--; + return callback(e.target.result); + } }; - return f.readAsText(blob, 'ISO-8859-1'); + return f.readAsText(blob); }; }).call(this); diff --git a/test/irc_utils_test.js b/test/irc_utils_test.js index dea61e15..5852d4ca 100644 --- a/test/irc_utils_test.js +++ b/test/irc_utils_test.js @@ -1,4 +1,3 @@ -var utf8 = '✓'; (function() { "use strict"; @@ -18,26 +17,20 @@ var utf8 = '✓'; cb = jasmine.createSpy('cb'); }); - it("calls the callback", function() { - ab = irc.util.arrayToArrayBuffer([65]); - fromSocketData(ab, cb); - waitsForArrayBufferConversion(); - return runs(function() { - expect(cb).toHaveBeenCalledWith('A'); - }); - }); - describe("handles encoding", function() { - [ - 'ISO 8859-1' - ].forEach(function(encoding) { + ['UTF-8', [0x61, 0xE2, 0x9C, 0x93], 'a✓'], + ['ISO 8859-1', [0x74, 0x73, 0x63, 0x68, 0xFC, 0xDF], 'tschüß'] + ].forEach(function(parts) { + var encoding = parts[0], + array = parts[1], + text = parts[2]; it(encoding, function() { - ab = irc.util.arrayToArrayBuffer([116, 115, 99, 104, 246]); + ab = irc.util.arrayToArrayBuffer(array); fromSocketData(ab, cb); waitsForArrayBufferConversion(); return runs(function() { - expect(cb).toHaveBeenCalledWith('tschö✓'); + expect(cb).toHaveBeenCalledWith(text); }); }); }); From 77ff18d5fbd04db354c2c6374841125222ea2e3b Mon Sep 17 00:00:00 2001 From: Timo Uhlmann Date: Thu, 4 Sep 2014 19:39:20 +0200 Subject: [PATCH 4/5] Dumping hex raw data --- package/bin/irc/irc.js | 4 +++- package/bin/irc/irc_util.js | 38 +++++++++++++++---------------------- 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/package/bin/irc/irc.js b/package/bin/irc/irc.js index 091939aa..5eba194e 100644 --- a/package/bin/irc/irc.js +++ b/package/bin/irc/irc.js @@ -232,7 +232,9 @@ this.data = this.data.slice(crlf + 1); dataView = new Uint8Array(this.data); _results.push(this.util.fromSocketData(line, function(lineStr) { - console.log('<=', "(" + _this.server + ")", lineStr); + console.groupCollapsed('<=', "(" + _this.server + ")", lineStr); + _this.util.dumpBuffer(line); + console.groupEnd(); return _this.onServerMessage(_this.util.parseCommand(lineStr)); })); } else { diff --git a/package/bin/irc/irc_util.js b/package/bin/irc/irc_util.js index 30f652ca..5034c4a7 100644 --- a/package/bin/irc/irc_util.js +++ b/package/bin/irc/irc_util.js @@ -144,6 +144,21 @@ return arrayBuffer; }; + exports.dumpBuffer = function(buffer) { + try { + var view, arr, hexes; + view = new Uint8Array(buffer); + arr = view.subarray(); + arr = Array.prototype.slice.call(arr); + hexes = arr.map(function(n) { + return (n < 16 ? '0' : '') + Number(n).toString(16).toUpperCase(); + }); + console.log(hexes.join(' ')); + } catch(e) { + console.err(e); + } + }; + function createBlob(src) { var BB = window.BlobBuilder || window.WebKitBlobBuilder; if (BB) { @@ -175,31 +190,8 @@ return f.readAsArrayBuffer(blob); }; - var dumpBuffer = function(buffer) { - console.groupCollapsed('<= Buffer [' + buffer.byteLength + ']'); - try { - var view, arr, hexes; - view = new Uint8Array(buffer); - arr = view.subarray(); - arr = Array.prototype.slice.call(arr); - hexes = arr.map(function(n) { - return Number(n).toString(16).toUpperCase(); - }); - // var offsets = new Array(16); - // for(var i=0; i<16; i++) { - // offsets[i] = Number(i).toString(16).toUpperCase(); - // } - // console.table([offsets, hexes]); - console.log(arr); - console.log(hexes.join(' ')); - } catch(e) { - console.err(e); - } - console.groupEnd(); - }; arrayBuffer2String = function(buf, callback) { - dumpBuffer(buf); var blob, f, hadReplacements; exports.arrayBufferConversionCount++; blob = createBlob(buf); From b5501b0441497efe51d95849a599cca95de3893d Mon Sep 17 00:00:00 2001 From: Timo Uhlmann Date: Thu, 4 Sep 2014 20:03:02 +0200 Subject: [PATCH 5/5] Added some more information to logging --- package/bin/irc/irc.js | 7 ++++++- package/bin/irc/irc_util.js | 3 +-- test/irc_utils_test.js | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/package/bin/irc/irc.js b/package/bin/irc/irc.js index 5eba194e..7301f321 100644 --- a/package/bin/irc/irc.js +++ b/package/bin/irc/irc.js @@ -231,8 +231,13 @@ var line = this.data.slice(0, cr ? crlf - 1 : crlf); this.data = this.data.slice(crlf + 1); dataView = new Uint8Array(this.data); - _results.push(this.util.fromSocketData(line, function(lineStr) { + _results.push(this.util.fromSocketData(line, function(lineStr, encoding) { console.groupCollapsed('<=', "(" + _this.server + ")", lineStr); + console.log( + encoding, + lineStr.length + ' characters', + line.byteLength + ' bytes' + ); _this.util.dumpBuffer(line); console.groupEnd(); return _this.onServerMessage(_this.util.parseCommand(lineStr)); diff --git a/package/bin/irc/irc_util.js b/package/bin/irc/irc_util.js index 5034c4a7..3af3e13a 100644 --- a/package/bin/irc/irc_util.js +++ b/package/bin/irc/irc_util.js @@ -199,12 +199,11 @@ hadReplacements = false; f.onload = function(e) { if(e.target.result.match(/\ufffd/) && !hadReplacements) { - console.log('Retrying with ISO-8859-1', e.target.result); hadReplacements = true; return f.readAsText(blob, 'ISO-8859-1'); } else { exports.arrayBufferConversionCount--; - return callback(e.target.result); + return callback(e.target.result, hadReplacements ? 'ISO-8859-1' : 'UTF-8'); } }; return f.readAsText(blob); diff --git a/test/irc_utils_test.js b/test/irc_utils_test.js index 5852d4ca..1a0d2591 100644 --- a/test/irc_utils_test.js +++ b/test/irc_utils_test.js @@ -20,7 +20,7 @@ describe("handles encoding", function() { [ ['UTF-8', [0x61, 0xE2, 0x9C, 0x93], 'a✓'], - ['ISO 8859-1', [0x74, 0x73, 0x63, 0x68, 0xFC, 0xDF], 'tschüß'] + ['ISO-8859-1', [0x74, 0x73, 0x63, 0x68, 0xFC, 0xDF], 'tschüß'] ].forEach(function(parts) { var encoding = parts[0], array = parts[1], @@ -30,7 +30,7 @@ fromSocketData(ab, cb); waitsForArrayBufferConversion(); return runs(function() { - expect(cb).toHaveBeenCalledWith(text); + expect(cb).toHaveBeenCalledWith(text, encoding); }); }); });