PLT-2816 Fixed handling of Unicode 8 emojis (#2924)

* Updated twemoji to properly recognize Unicode 8.0 emojis

* Updated unicode emoji parser to only render emojis we support as images

* Corrected filename for South African flag emoji

* Added Mattermost emoticons!

* Added additional emoticons to test files

webapp/components/suggestion/emoticon_provider.jsx

@@ -55,7 +55,7 @@ export default class EmoticonProvider {
 
             const matched = [];
 
-            for (const [name, emoticon] of Emoticons.emoticons) {
+            for (const [name, emoticon] of Emoticons.getEmoticonsByName()) {
                 if (name.indexOf(partialName) !== -1) {
                     matched.push(emoticon);
 

webapp/package.json

@@ -29,7 +29,7 @@
     "react-router": "2.0.1",
     "react-textarea-autosize": "3.3.0",
     "superagent": "1.8.3",
-    "twemoji": "1.4.1",
+    "twemoji": "2.0.5",
     "velocity-animate": "1.2.3"
   },
   "devDependencies": {

webapp/utils/emoji.json

@@ -8181,6 +8181,14 @@
   , "tags": [
     ]
   }
+, {
+    "aliases": [
+      "mm",
+      "mattermost"
+    ]
+  , "tags": [
+    ]
+  }
 , {
     "aliases": [
       "basecamp"

webapp/utils/emoticons.jsx

@@ -29,10 +29,12 @@ const emoticonPatterns = {
     thumbsdown: /(^|\s)(:\-1:)(?=$|\s)/g // :-1:
 };
 
-export const emoticons = initializeEmoticons();
+let emoticonsByName;
+let emoticonsByCodePoint;
 
 function initializeEmoticons() {
-    const emoticonMap = new Map();
+    emoticonsByName = new Map();
+    emoticonsByCodePoint = new Set();
 
     for (const emoji of emojis) {
         const unicode = emoji.emoji;
@@ -40,6 +42,8 @@ function initializeEmoticons() {
         let filename = '';
         if (unicode) {
             // this is a unicode emoji so the character code determines the file name
+            let codepoint = '';
+
             for (let i = 0; i < unicode.length; i += 2) {
                 const code = fixedCharCodeAt(unicode, i);
 
@@ -50,25 +54,26 @@ function initializeEmoticons() {
 
                 // some emoji (such as country flags) span multiple unicode characters
                 if (i !== 0) {
-                    filename += '-';
+                    codepoint += '-';
                 }
 
-                filename += pad(code.toString(16));
+                codepoint += pad(code.toString(16));
             }
+
+            filename = codepoint;
+            emoticonsByCodePoint.add(codepoint);
         } else {
             // this isn't a unicode emoji so the first alias determines the file name
             filename = emoji.aliases[0];
         }
 
         for (const alias of emoji.aliases) {
-            emoticonMap.set(alias, {
+            emoticonsByName.set(alias, {
                 alias,
                 path: getImagePathForEmoticon(filename)
             });
         }
     }
-
-    return emoticonMap;
 }
 
 // Pads a hexadecimal number with zeroes to be at least 4 digits long
@@ -110,14 +115,30 @@ function fixedCharCodeAt(str, idx = 0) {
     return code;
 }
 
+export function getEmoticonsByName() {
+    if (!emoticonsByName) {
+        initializeEmoticons();
+    }
+
+    return emoticonsByName;
+}
+
+export function getEmoticonsByCodePoint() {
+    if (!emoticonsByCodePoint) {
+        initializeEmoticons();
+    }
+
+    return emoticonsByCodePoint;
+}
+
 export function handleEmoticons(text, tokens) {
     let output = text;
 
     function replaceEmoticonWithToken(fullMatch, prefix, matchText, name) {
-        if (emoticons.has(name)) {
+        if (getEmoticonsByName().has(name)) {
             const index = tokens.size;
             const alias = `MM_EMOTICON${index}`;
-            const path = emoticons.get(name).path;
+            const path = getEmoticonsByName().get(name).path;
 
             tokens.set(alias, {
                 value: `<img align="absmiddle" alt="${matchText}" class="emoticon" src="${path}" title="${matchText}" />`,
@@ -141,6 +162,6 @@ export function handleEmoticons(text, tokens) {
     return output;
 }
 
-function getImagePathForEmoticon(name) {
+export function getImagePathForEmoticon(name) {
     return Constants.EMOJI_PATH + '/' + name + '.png';
 }

webapp/utils/text_formatting.jsx

@@ -60,17 +60,25 @@ export function doFormatText(text, options) {
         output = highlightCurrentMentions(output, tokens);
     }
 
-    // reinsert tokens with formatted versions of the important words and phrases
-    output = replaceTokens(output, tokens);
-
     if (!('emoticons' in options) || options.emoticon) {
         output = twemoji.parse(output, {
             className: 'emoticon',
             base: '',
-            folder: Constants.EMOJI_PATH
+            folder: Constants.EMOJI_PATH,
+            callback: (icon, twemojiOptions) => {
+                if (!Emoticons.getEmoticonsByCodePoint().has(icon)) {
+                    // just leave the unicode characters and hope the browser can handle it
+                    return null;
+                }
+
+                return ''.concat(twemojiOptions.base, twemojiOptions.size, '/', icon, twemojiOptions.ext);
+            }
         });
     }
 
+    // reinsert tokens with formatted versions of the important words and phrases
+    output = replaceTokens(output, tokens);
+
     return output;
 }