Replace emojione with twemoji + emojibase

package.json

@@ -65,7 +65,8 @@
     "classnames": "^2.1.2",
     "commonmark": "^0.28.1",
     "counterpart": "^0.18.0",
-    "emojione": "2.2.7",
+    "emojibase-data": "^4.0.0",
+    "emojibase-regex": "^3.0.0",
     "file-saver": "^1.3.3",
     "filesize": "3.5.6",
     "flux": "2.1.1",

res/css/_common.scss

@@ -32,6 +32,10 @@ body {
     margin: 0px;
 }
 
+code {
+    font-family: $monospace-font-family;
+}
+
 .error, .warning {
     color: $warning-color;
 }
@@ -445,15 +449,6 @@ textarea {
     background-color: $primary-bg-color;
 }
 
-.mx_emojione {
-    height: 1em;
-    vertical-align: middle;
-}
-
-.mx_emojione_selected {
-    background-color: $accent-color;
-}
-
 ::-moz-selection {
     background-color: $accent-color;
     color: $selection-fg-color;

res/css/views/rooms/_EventTile.scss

@@ -108,7 +108,7 @@ limitations under the License.
 /* HACK to override line-height which is already marked important elsewhere */
 .mx_EventTile_bigEmoji.mx_EventTile_bigEmoji {
     font-size: 48px ! important;
-    line-height: 48px ! important;
+    line-height: 52px ! important;
 }
 
 /* this is used for the tile for the event which is selected via the URL.
@@ -149,8 +149,7 @@ limitations under the License.
 }
 
 .mx_EventTile_sending .mx_UserPill,
-.mx_EventTile_sending .mx_RoomPill,
-.mx_EventTile_sending .mx_emojione {
+.mx_EventTile_sending .mx_RoomPill {
     opacity: 0.5;
 }
 
@@ -412,6 +411,7 @@ limitations under the License.
 
 .mx_EventTile_content .markdown-body {
     pre, code {
+        font-family: $monospace-font-family ! important;
         // deliberate constants as we're behind an invert filter
         color: #333;
     }

res/themes/light/css/_fonts.scss

@@ -15,22 +15,22 @@
 /* the 'src' links are relative to the bundle.css, which is in a subdirectory.
  */
 @font-face {
-  font-family: 'Nunito';
-  font-style: normal;
-  font-weight: 400;
-  src: url('$(res)/fonts/Nunito/Nunito-Regular.ttf') format('truetype');
+    font-family: 'Nunito';
+    font-style: normal;
+    font-weight: 400;
+    src: url('$(res)/fonts/Nunito/Nunito-Regular.ttf') format('truetype');
 }
 @font-face {
-  font-family: 'Nunito';
-  font-style: normal;
-  font-weight: 600;
-  src: url('$(res)/fonts/Nunito/Nunito-SemiBold.ttf') format('truetype');
+    font-family: 'Nunito';
+    font-style: normal;
+    font-weight: 600;
+    src: url('$(res)/fonts/Nunito/Nunito-SemiBold.ttf') format('truetype');
 }
 @font-face {
-  font-family: 'Nunito';
-  font-style: normal;
-  font-weight: 700;
-  src: url('$(res)/fonts/Nunito/Nunito-Bold.ttf') format('truetype');
+    font-family: 'Nunito';
+    font-style: normal;
+    font-weight: 700;
+    src: url('$(res)/fonts/Nunito/Nunito-Bold.ttf') format('truetype');
 }
 
 /*
@@ -51,3 +51,13 @@
     font-weight: 700;
     font-style: normal;
 }
+
+/* a COLR/CPAL version of Twemoji used for consistent cross-browser emoji
+ * taken from https://github.com/mozilla/twemoji-colr
+ * using the fix from https://github.com/mozilla/twemoji-colr/issues/50 to
+ * work on macOS
+ */
+@font-face {
+    font-family: "Twemoji Mozilla";
+    src: url('$(res)/fonts/Twemoji_Mozilla/TwemojiMozilla.woff2') format('woff2');
+}

res/themes/light/css/_light.scss

@@ -1,10 +1,15 @@
 // XXX: check this?
 /* Nunito lacks combining diacritics, so these will fall through
    to the next font.  Helevetica's diacritics however do not combine
-   nicely with Open Sans (on OSX, at least) and result in a huge
-   horizontal mess.  Arial empirically gets it right, hence prioritising
-   Arial here. */
-$font-family: 'Nunito', Arial, Helvetica, Sans-Serif;
+   nicely (on OSX, at least) and result in a huge horizontal mess.
+   Arial empirically gets it right, hence prioritising Arial here. */
+/* We fall through to Twemoji for emoji rather than falling through
+   to native Emoji fonts (if any) to ensure cross-browser consistency */
+$font-family: Nunito, 'Twemoji Mozilla', Arial, Helvetica, Sans-Serif;
+
+// XXX: In theory this should be Fira, but it's a bit ugly.
+// TODO: make it consistent cross-browser
+$monospace-font-family: Consolas, 'Liberation Mono', Courier, 'Twemoji Mozilla', monospace;
 
 // unified palette
 // try to use these colors when possible

scripts/emoji-data-strip.js

@@ -1,28 +1,29 @@
 #!/usr/bin/env node
-const EMOJI_DATA = require('emojione/emoji.json');
-const EMOJI_SUPPORTED = Object.keys(require('emojione').emojioneList);
+
+// This generates src/stripped-emoji.json as used by the EmojiProvider autocomplete
+// provider.
+
+const EMOJIBASE = require('emojibase-data/en/compact.json');
+
 const fs = require('fs');
 
-const output = Object.keys(EMOJI_DATA).map(
-    (key) => {
-        const datum = EMOJI_DATA[key];
+const output = EMOJIBASE.map(
+    (datum) => {
         const newDatum = {
-            name: datum.name,
-            shortname: datum.shortname,
-            category: datum.category,
-            emoji_order: datum.emoji_order,
+            name: datum.annotation,
+            shortname: `:${datum.shortcodes[0]}:`,
+            category: datum.group,
+            emoji_order: datum.order,
         };
-        if (datum.aliases.length > 0) {
-            newDatum.aliases = datum.aliases;
+        if (datum.shortcodes.length > 1) {
+            newDatum.aliases = datum.shortcodes.slice(1).map(s => `:${s}:`);
         }
-        if (datum.aliases_ascii.length > 0) {
-            newDatum.aliases_ascii = datum.aliases_ascii;
+        if (datum.emoticon) {
+            newDatum.aliases_ascii = [ datum.emoticon ];
         }
         return newDatum;
     }
-).filter((datum) => {
-    return EMOJI_SUPPORTED.includes(datum.shortname);
-});
+);
 
 // Write to a file in src. Changes should be checked into git. This file is copied by
 // babel using --copy-files

src/HtmlUtils.js

@@ -27,22 +27,18 @@ import linkifyMatrix from './linkify-matrix';
 import _linkifyElement from 'linkifyjs/element';
 import _linkifyString from 'linkifyjs/string';
 import escape from 'lodash/escape';
-import emojione from 'emojione';
 import classNames from 'classnames';
 import MatrixClientPeg from './MatrixClientPeg';
 import url from 'url';
 
-linkifyMatrix(linkify);
+import EMOJIBASE from 'emojibase-data/en/compact.json';
+import EMOJIBASE_REGEX from 'emojibase-regex';
 
-emojione.imagePathSVG = 'emojione/svg/';
-// Store PNG path for displaying many flags at once (for increased performance over SVG)
-emojione.imagePathPNG = 'emojione/png/';
-// Use SVGs for emojis
-emojione.imageType = 'svg';
+linkifyMatrix(linkify);
 
 // Anything outside the basic multilingual plane will be a surrogate pair
 const SURROGATE_PAIR_PATTERN = /([\ud800-\udbff])([\udc00-\udfff])/;
-// And there a bunch more symbol characters that emojione has within the
+// And there a bunch more symbol characters that emojibase has within the
 // BMP, so this includes the ranges from 'letterlike symbols' to
 // 'miscellaneous symbols and arrows' which should catch all of them
 // (with plenty of false positives, but that's OK)
@@ -54,15 +50,15 @@ const ZWJ_REGEX = new RegExp("\u200D|\u2003", "g");
 // Regex pattern for whitespace characters
 const WHITESPACE_REGEX = new RegExp("\\s", "g");
 
-// And this is emojione's complete regex
-const EMOJI_REGEX = new RegExp(emojione.unicodeRegexp+"+", "gi");
+const BIGEMOJI_REGEX = new RegExp(`^(${EMOJIBASE_REGEX.source})+$`, 'i');
+
 const COLOR_REGEX = /^#[0-9a-fA-F]{6}$/;
 
 const PERMITTED_URL_SCHEMES = ['http', 'https', 'ftp', 'mailto', 'magnet'];
 
 /*
  * Return true if the given string contains emoji
- * Uses a much, much simpler regex than emojione's so will give false
+ * Uses a much, much simpler regex than emojibase's so will give false
  * positives, but useful for fast-path testing strings to see if they
  * need emojification.
  * unicodeToImage uses this function.
@@ -71,76 +67,30 @@ export function containsEmoji(str) {
     return SURROGATE_PAIR_PATTERN.test(str) || SYMBOL_PATTERN.test(str);
 }
 
-/* modified from https://github.com/Ranks/emojione/blob/master/lib/js/emojione.js
- * because we want to include emoji shortnames in title text
- */
-function unicodeToImage(str, addAlt) {
-    if (addAlt === undefined) addAlt = true;
-
-    let replaceWith; let unicode; let short; let fname;
-    const mappedUnicode = emojione.mapUnicodeToShort();
-
-    str = str.replace(emojione.regUnicode, function(unicodeChar) {
-        if ( (typeof unicodeChar === 'undefined') || (unicodeChar === '') || (!(unicodeChar in emojione.jsEscapeMap)) ) {
-            // if the unicodeChar doesnt exist just return the entire match
-            return unicodeChar;
-        } else {
-            // get the unicode codepoint from the actual char
-            unicode = emojione.jsEscapeMap[unicodeChar];
-
-            short = mappedUnicode[unicode];
-            fname = emojione.emojioneList[short].fname;
-
-            // depending on the settings, we'll either add the native unicode as the alt tag, otherwise the shortname
-            const title = mappedUnicode[unicode];
-
-            if (addAlt) {
-                const alt = (emojione.unicodeAlt) ? emojione.convert(unicode.toUpperCase()) : mappedUnicode[unicode];
-                replaceWith = `<img class="mx_emojione" title="${title}" alt="${alt}" src="${emojione.imagePathSVG}${fname}.svg${emojione.cacheBustParam}"/>`;
-            } else {
-                replaceWith = `<img class="mx_emojione" src="${emojione.imagePathSVG}${fname}.svg${emojione.cacheBustParam}"/>`;
-            }
-            return replaceWith;
-        }
-    });
-
-    return str;
-}
-
 /**
  * Returns the shortcode for an emoji character.
  *
  * @param {String} char The emoji character
  * @return {String} The shortcode (such as :thumbup:)
  */
-export function unicodeToShort(char) {
-    const unicode = emojione.jsEscapeMap[char];
-    return emojione.mapUnicodeToShort()[unicode];
+export function unicodeToShortcode(char) {
+    const data = EMOJIBASE.find(e => e.unicode === char);
+    return (data && data.shortcodes ? `:${data.shortcodes[0]}:` : '');
 }
 
 /**
- * Given one or more unicode characters (represented by unicode
- * character number), return an image node with the corresponding
- * emoji.
+ * Returns the unicode character for an emoji shortcode
  *
- * @param alt {string} String to use for the image alt text
- * @param useSvg {boolean} Whether to use SVG image src. If False, PNG will be used.
- * @param unicode {integer} One or more integers representing unicode characters
- * @returns A img node with the corresponding emoji
+ * @param {String} shortcode The shortcode (such as :thumbup:)
+ * @return {String} The emoji character; null if none exists
  */
-export function charactersToImageNode(alt, useSvg, ...unicode) {
-    const fileName = unicode.map((u) => {
-        return u.toString(16);
-    }).join('-');
-    const path = useSvg ? emojione.imagePathSVG : emojione.imagePathPNG;
-    const fileType = useSvg ? 'svg' : 'png';
-    return <img
-        alt={alt}
-        src={`${path}${fileName}.${fileType}${emojione.cacheBustParam}`}
-    />;
+export function shortcodeToUnicode(shortcode) {
+    shortcode = shortcode.slice(1, shortcode.length - 1);
+    const data = EMOJIBASE.find(e => e.shortcodes && e.shortcodes.includes(shortcode));
+    return data ? data.unicode : null;
 }
 
-export function processHtmlForSending(html: string): string {
+export function processHtmlForSending   (html: string): string {
     const contentDiv = document.createElement('div');
     contentDiv.innerHTML = html;
 
@@ -444,13 +394,10 @@ class TextHighlighter extends BaseHighlighter {
  * opts.disableBigEmoji: optional argument to disable the big emoji class.
  * opts.stripReplyFallback: optional argument specifying the event is a reply and so fallback needs removing
  * opts.returnString: return an HTML string rather than JSX elements
- * opts.emojiOne: optional param to do emojiOne (default true)
  * opts.forComposerQuote: optional param to lessen the url rewriting done by sanitization, for quoting into composer
  */
 export function bodyToHtml(content, highlights, opts={}) {
     const isHtmlMessage = content.format === "org.matrix.custom.html" && content.formatted_body;
-
-    const doEmojiOne = opts.emojiOne === undefined ? true : opts.emojiOne;
     let bodyHasEmoji = false;
 
     let sanitizeParams = sanitizeHtmlParams;
@@ -481,28 +428,12 @@ export function bodyToHtml(content, highlights, opts={}) {
         if (opts.stripReplyFallback && formattedBody) formattedBody = ReplyThread.stripHTMLReply(formattedBody);
         strippedBody = opts.stripReplyFallback ? ReplyThread.stripPlainReply(content.body) : content.body;
 
-        if (doEmojiOne) {
-            bodyHasEmoji = containsEmoji(isHtmlMessage ? formattedBody : content.body);
-        }
+        bodyHasEmoji = containsEmoji(isHtmlMessage ? formattedBody : content.body);
 
         // Only generate safeBody if the message was sent as org.matrix.custom.html
         if (isHtmlMessage) {
             isDisplayedWithHtml = true;
             safeBody = sanitizeHtml(formattedBody, sanitizeParams);
-        } else {
-            // ... or if there are emoji, which we insert as HTML alongside the
-            // escaped plaintext body.
-            if (bodyHasEmoji) {
-                isDisplayedWithHtml = true;
-                safeBody = sanitizeHtml(escape(strippedBody), sanitizeParams);
-            }
-        }
-
-        // An HTML message with emoji
-        //  or a plaintext message with emoji that was escaped and sanitized into
-        //  HTML.
-        if (bodyHasEmoji) {
-            safeBody = unicodeToImage(safeBody);
         }
     } finally {
         delete sanitizeParams.textFilter;
@@ -514,7 +445,6 @@ export function bodyToHtml(content, highlights, opts={}) {
 
     let emojiBody = false;
     if (!opts.disableBigEmoji && bodyHasEmoji) {
-        EMOJI_REGEX.lastIndex = 0;
         let contentBodyTrimmed = strippedBody !== undefined ? strippedBody.trim() : '';
 
         // Ignore spaces in body text. Emojis with spaces in between should
@@ -526,12 +456,14 @@ export function bodyToHtml(content, highlights, opts={}) {
         // presented as large.
         contentBodyTrimmed = contentBodyTrimmed.replace(ZWJ_REGEX, '');
 
-        const match = EMOJI_REGEX.exec(contentBodyTrimmed);
-        emojiBody = match && match[0] && match[0].length === contentBodyTrimmed.length
+        const match = BIGEMOJI_REGEX.exec(contentBodyTrimmed);
+        emojiBody = match && match[0] && match[0].length === contentBodyTrimmed.length &&
                     // Prevent user pills expanding for users with only emoji in
                     // their username
-                    && (content.formatted_body == undefined
-		        || !content.formatted_body.includes("https://matrix.to/"));
+                    (
+                        content.formatted_body == undefined ||
+		                !content.formatted_body.includes("https://matrix.to/")
+                    );
     }
 
     const className = classNames({
@@ -545,12 +477,6 @@ export function bodyToHtml(content, highlights, opts={}) {
         <span className={className} dir="auto">{ strippedBody }</span>;
 }
 
-export function emojifyText(text, addAlt) {
-    return {
-        __html: unicodeToImage(escape(text), addAlt),
-    };
-}
-
 /**
  * Linkifies the given string. This is a wrapper around 'linkifyjs/string'.
  *