From 35efea5825ad83e986c9b7c3587d35aa70a1bb9f Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Fri, 12 Nov 2021 02:27:11 +1300 Subject: [PATCH] feat(unicode): strip additional rune ranges (#1575) --- helper/unicode.js | 2 ++ test/unit/helper/unicode.js | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/helper/unicode.js b/helper/unicode.js index e0fdaab80..b8d8cea6d 100644 --- a/helper/unicode.js +++ b/helper/unicode.js @@ -73,10 +73,12 @@ const MISC_UNSUPPORTED_SYMBOLS = regenerate() .addRange(0x1D400, 0x1D7FF) // Mathematical Alphanumeric Symbols (1D400-1D7FF) // emojis + .addRange(0x1F1E6, 0x1F1FF) // Regional indicator symbol (1F1E6-1F1FF) .addRange(0x1F300, 0x1F5FF) // Miscellaneous Symbols and Pictographs (1F300-1F5FF) .addRange(0x1F3FB, 0x1F3FF) // Emoji Modifier Fitzpatrick (skin tones) (1F3FB–1F3FF) .addRange(0x1F600, 0x1F64F) // Emoticons (1F600–1F64F) .addRange(0x1F680, 0x1F6FF) // Transport and Map Symbols (1F680-1F6FF) + .addRange(0x1F780, 0x1F7FF) // Geometric Shapes Extended (1F780-1F7FF) .addRange(0x1F900, 0x1F9FF) // Supplemental Symbols and Pictographs (1F900-1F9FF) .toRegExp('g'); diff --git a/test/unit/helper/unicode.js b/test/unit/helper/unicode.js index eafde8f8d..37ff06fea 100644 --- a/test/unit/helper/unicode.js +++ b/test/unit/helper/unicode.js @@ -47,6 +47,13 @@ module.exports.tests.normalize = function (test) { t.equal(norm('πŸ‘©β€β€οΈβ€πŸ‘©'), '', 'complex emoji ZWJ sequence (6 codepoints)'); t.end(); }); + + // https://github.com/pelias/api/issues/1574 + test('normalize: issue 1574', function (t) { + const input = 'ASDFJK БВРАНЫ Π‘Π«Π›Π˜ КВО БЛУШАЕВ ΠœΠ•ΠΠ―Π€1πŸ’’πŸŒ‹πŸš­πŸš±β¬†οΈβ†—οΈβž‘οΈβ†˜οΈβ¬‡οΈβ†™οΈβ¬…οΈβ†–οΈβ†•οΈβ†”οΈβ†©β†ͺβ€΄οΈπŸ‘―β€β™‚οΈπŸ‡°πŸ‡Ώβ€΅οΈπŸ”ƒπŸ”„πŸ›βš›πŸ•‰βœ‘β˜Έβ™β˜―οΈβ˜Έβœ‘πŸ•‰πŸ›βš›πŸ”…πŸ”†πŸ”΄πŸŸ πŸŸ‘πŸŸ’ΠΠ—Π•Π’Π˜ Π› К Π’Πž УНВВА854Π£32Π™1πŸ”΅πŸŸ£πŸŸ€βšͺ⚫⚫πŸŸ₯πŸŸ§πŸŸ¨πŸ”ΆοΈπŸŸ©πŸŸ¦πŸŸͺπŸŸ«β¬›β¬œβ—Όβ—»β—Ύβ—½πŸ’ '; // jshint ignore:line + t.equal(norm(input), 'ASDFJK БВРАНЫ Π‘Π«Π›Π˜ КВО БЛУШАЕВ ΠœΠ•ΠΠ―Π€1ΠΠ—Π•Π’Π˜ Π› К Π’Πž УНВВА854Π£32Π™1', 'issue 1574'); + t.end(); + }); }; module.exports.all = function (tape, common) {