// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. /* * This script will generate the emoji files for both the webapp and server to use emojis from 'emoji-datasource' npm package * It will generate the following files: * '/webapp/channels/src/utils/emoji.ts' * '/webapp/channels/src/sass/components/_emojisprite.scss' * '/webapp/channels/src/utils/emoji.json' * '/server/public/model/emoji_data.go', * * For help on how to use this script, run: * npm run make-emojis -- --help */ import * as fs from 'node:fs'; import * as fsPromise from 'node:fs/promises'; import path from 'node:path'; import * as url from 'node:url'; import chalk from 'chalk'; import jsonCategories from 'emoji-datasource/categories.json' assert {type: 'json'}; import jsonData from 'emoji-datasource/emoji.json' assert {type: 'json'}; import yargs from 'yargs'; import additionalShortnames from './additional_shortnames.json' assert {type: 'json'}; const EMOJI_SIZE = 64; const EMOJI_SIZE_PADDED = EMOJI_SIZE + 2; // 1px per side const EMOJI_DEFAULT_SKIN = 'default'; const endResults = []; const currentDir = path.dirname(url.fileURLToPath(import.meta.url)); const rootDir = path.resolve(currentDir, '..', '..', '..', '..'); const serverRootDir = path.resolve(rootDir, 'server'); const webappRootDir = path.resolve(rootDir, 'webapp'); const argv = yargs(process.argv.slice(1)). scriptName('make-emojis'). usage('Usage : npm run $0 -- [args]'). example('npm run $0 -- --excluded-emoji-file ./excludedEmojis.txt', 'removes mentioned emojis from the app'). option('excluded-emoji-file', { description: 'Path to a file containing emoji short names to exclude', type: 'string', }). help(). epilog('Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.'). argv; const argsExcludedEmojiFile = argv['excluded-emoji-file']; function log(level = '', text) { if (level === 'info') { // eslint-disable-next-line no-console console.log(chalk.cyan(`[INFO]: ${text}`)); } else if (level === 'warn') { // eslint-disable-next-line no-console console.log(chalk.yellow(`[WARN]: ${text}`)); } else if (level === 'error') { // eslint-disable-next-line no-console console.error(chalk.red(`[ERRO]: ${text}`)); } else if (level === 'success') { // eslint-disable-next-line no-console console.log(chalk.green(`[SUCC]: ${text}`)); } else { // eslint-disable-next-line no-console console.log(text); } } function writeToFileAndPrint(fileName, filePath, data) { const promise = fsPromise.writeFile(filePath, data, writeOptions); promise.then(() => { log('info', `"${fileName}" generated successfully in "${filePath}"`); }).catch((err) => { log('error', `Failed to generate "${fileName}": ${err}`); }); return promise; } function copyFileAndPrint(source, destination, fileName, print = true) { const promise = fsPromise.copyFile(source, destination); promise.then(() => { if (print) { log('info', `"${fileName}" copied successfully to "${destination}"`); } }).catch((err) => { log('error', `Failed to copy "${fileName}": ${err}`); }); return promise; } function normalizeCategoryName(category) { return category.toLowerCase().replace(' & ', '-'); } // Copy emoji images from 'emoji-datasource-apple' const emojiDataSourceDir = path.resolve(webappRootDir, `node_modules/emoji-datasource-apple/img/apple/${EMOJI_SIZE}/`); const emojiImagesDir = path.resolve(webappRootDir, 'channels', 'src', 'images', 'emoji'); const readDirPromise = fsPromise.readdir(emojiDataSourceDir); endResults.push(readDirPromise); readDirPromise.then((images) => { for (const imageFile of images) { endResults.push(copyFileAndPrint(path.join(emojiDataSourceDir, imageFile), path.join(emojiImagesDir, imageFile), imageFile, false)); } }); // Missing emojis from Apple emoji set ["medical_symbol", "male_sign", and "female_sign"] // @see https://github.com/iamcal/emoji-data#image-sources const missingEmojis = ['2640-fe0f', '2642-fe0f', '2695-fe0f']; // Copy the missing from google emoji set const missingEmojiDataSourceDir = path.resolve(webappRootDir, `node_modules/emoji-datasource-google/img/google/${EMOJI_SIZE}/`); const readMissingDirPromise = fsPromise.readdir(missingEmojiDataSourceDir); endResults.push(readMissingDirPromise); readMissingDirPromise.then(() => { for (const missingEmoji of missingEmojis) { endResults.push( copyFileAndPrint(path.join(missingEmojiDataSourceDir, `${missingEmoji}.png`), path.join(emojiImagesDir, `${missingEmoji}.png`), `Missed ${missingEmoji}`)); } }); // Copy mattermost emoji image const webappImagesDir = path.resolve(webappRootDir, 'channels', 'src', 'images'); endResults.push(copyFileAndPrint(path.resolve(webappImagesDir, 'icon64x64.png'), path.resolve(webappImagesDir, 'emoji/mattermost.png'), 'mattermost-emoji')); const sheetSource = path.resolve(webappRootDir, `node_modules/emoji-datasource-apple/img/apple/sheets/${EMOJI_SIZE}.png`); const sheetAbsoluteFile = path.resolve(webappRootDir, 'channels', 'src', 'images/emoji-sheets/apple-sheet.png'); const sheetFile = 'images/emoji-sheets/apple-sheet.png'; // Copy sheet image endResults.push(copyFileAndPrint(sheetSource, sheetAbsoluteFile, 'emoji-sheet')); // we'll load it as a two dimensional array so we can generate a Map out of it const emojiIndicesByAlias = []; const emojiIndicesByUnicode = []; const emojiIndicesByCategory = new Map(); const emojiIndicesByCategoryAndSkin = new Map(); const emojiIndicesByCategoryNoSkin = new Map(); const categoryNamesSet = new Set(); const emojiImagesByAlias = []; const emojiFilePositions = new Map(); const skinCodes = { '1F3FB': 'light_skin_tone', '1F3FC': 'medium_light_skin_tone', '1F3FD': 'medium_skin_tone', '1F3FE': 'medium_dark_skin_tone', '1F3FF': 'dark_skin_tone', default: 'default', }; const skinNames = { '1F3FB': 'LIGHT SKIN TONE', '1F3FC': 'MEDIUM LIGHT SKIN TONE', '1F3FD': 'MEDIUM SKIN TONE', '1F3FE': 'MEDIUM DARK SKIN TONE', '1F3FF': 'DARK SKIN TONE', }; const control = new AbortController(); const writeOptions = { encoding: 'utf8', signal: control.signal, }; // Extract excluded emoji shortnames as an array const excludedEmoji = []; if (argsExcludedEmojiFile) { fs.readFileSync(path.normalize(argsExcludedEmojiFile), 'utf-8').split(/\r?\n/).forEach((line) => { excludedEmoji.push(line); }); log('warn', `The following emoji(s) will be excluded from the webapp: \n${excludedEmoji}\n`); } // Remove unwanted emoji const filteredEmojiJson = jsonData.filter((element) => !excludedEmoji.some((e) => element.short_names.includes(e))); function generateEmojiSkinVariations(emoji) { if (!emoji.skin_variations) { return []; } return Object.keys(emoji.skin_variations).map((skinCode) => { // if skin codes ever change this will produce a null_light_skin_tone const skins = skinCode.split('-'); const skinShortName = skins.map((code) => skinCodes[code]).join('_'); const skinName = skins.map((code) => skinNames[code]).join(', '); const variation = {...emoji.skin_variations[skinCode]}; variation.short_name = `${emoji.short_name}_${skinShortName}`; variation.short_names = emoji.short_names.map((alias) => `${alias}_${skinShortName}`); variation.name = `${emoji.name}: ${skinName}`; variation.category = emoji.category; variation.skins = skins; return variation; }); } // populate skin tones as full emojis const fullEmoji = [...filteredEmojiJson]; filteredEmojiJson.forEach((emoji) => { const variations = generateEmojiSkinVariations(emoji); fullEmoji.push(...variations); }); // add old shortnames to maintain backwards compatibility with gemoji fullEmoji.forEach((emoji) => { if (emoji.short_name in additionalShortnames) { emoji.short_names.push(...additionalShortnames[emoji.short_name]); } }); // add built-in custom emojis fullEmoji.push({ id: 'mattermost', name: 'Mattermost', unified: '', image: 'mattermost.png', short_name: 'mattermost', short_names: ['mattermost'], category: 'custom', }); fullEmoji.sort((emojiA, emojiB) => emojiA.sort_order - emojiB.sort_order); function addIndexToMap(emojiMap, key, ...indexes) { const newList = emojiMap.get(key) || []; newList.push(...indexes); emojiMap.set(key, newList); } const skinset = new Set(); fullEmoji.forEach((emoji, index) => { if (emoji.unified) { emojiIndicesByUnicode.push([emoji.unified.toLowerCase(), index]); } const safeCat = normalizeCategoryName(emoji.category); emoji.category = safeCat; addIndexToMap(emojiIndicesByCategory, safeCat, index); if (emoji.skins || emoji.skin_variations) { const skin = (emoji.skins && emoji.skins[0]) || EMOJI_DEFAULT_SKIN; skinset.add(skin); const categoryBySkin = emojiIndicesByCategoryAndSkin.get(skin) || new Map(); addIndexToMap(categoryBySkin, safeCat, index); emojiIndicesByCategoryAndSkin.set(skin, categoryBySkin); } else { addIndexToMap(emojiIndicesByCategoryNoSkin, safeCat, index); } categoryNamesSet.add(safeCat); emojiIndicesByAlias.push(...emoji.short_names.map((alias) => [alias, index])); const file = emoji.image.split('.')[0]; emoji.fileName = emoji.image; emoji.image = file; if (emoji.category !== 'custom') { let x = emoji.sheet_x * EMOJI_SIZE_PADDED; if (x !== 0) { x += 'px'; } let y = emoji.sheet_y * EMOJI_SIZE_PADDED; if (y !== 0) { y += 'px'; } emojiFilePositions.set(file, `-${x} -${y};`); } emojiImagesByAlias.push(...emoji.short_names.map((alias) => `"${alias}": "${file}"`)); }); function trimPropertiesFromEmoji(emoji) { if (Object.hasOwn(emoji, 'non_qualified')) { Reflect.deleteProperty(emoji, 'non_qualified'); } if (Object.hasOwn(emoji, 'docomo')) { Reflect.deleteProperty(emoji, 'docomo'); } if (Object.hasOwn(emoji, 'au')) { Reflect.deleteProperty(emoji, 'au'); } if (Object.hasOwn(emoji, 'softbank')) { Reflect.deleteProperty(emoji, 'softbank'); } if (Object.hasOwn(emoji, 'google')) { Reflect.deleteProperty(emoji, 'google'); } if (Object.hasOwn(emoji, 'sheet_x')) { Reflect.deleteProperty(emoji, 'sheet_x'); } if (Object.hasOwn(emoji, 'sheet_y')) { Reflect.deleteProperty(emoji, 'sheet_y'); } if (Object.hasOwn(emoji, 'added_in')) { Reflect.deleteProperty(emoji, 'added_in'); } if (Object.hasOwn(emoji, 'has_img_apple')) { Reflect.deleteProperty(emoji, 'has_img_apple'); } if (Object.hasOwn(emoji, 'has_img_google')) { Reflect.deleteProperty(emoji, 'has_img_google'); } if (Object.hasOwn(emoji, 'has_img_twitter')) { Reflect.deleteProperty(emoji, 'has_img_twitter'); } if (Object.hasOwn(emoji, 'has_img_facebook')) { Reflect.deleteProperty(emoji, 'has_img_facebook'); } if (Object.hasOwn(emoji, 'source_index')) { Reflect.deleteProperty(emoji, 'source_index'); } if (Object.hasOwn(emoji, 'sort_order')) { Reflect.deleteProperty(emoji, 'sort_order'); } if (Object.hasOwn(emoji, 'subcategory')) { Reflect.deleteProperty(emoji, 'subcategory'); } if (Object.hasOwn(emoji, 'image')) { Reflect.deleteProperty(emoji, 'image'); } if (Object.hasOwn(emoji, 'fileName')) { Reflect.deleteProperty(emoji, 'fileName'); } return emoji; } // Removed properties that are not needed for the webapp const trimmedDownEmojis = fullEmoji.map((emoji) => trimPropertiesFromEmoji(emoji)); // Create the emoji.json file const webappUtilsEmojiDir = path.resolve(webappRootDir, 'channels', 'src', 'utils', 'emoji.json'); endResults.push(writeToFileAndPrint('emoji.json', webappUtilsEmojiDir, JSON.stringify(trimmedDownEmojis, null, 4))); const categoryList = Object.keys(jsonCategories).filter((item) => item !== 'Component').map(normalizeCategoryName); const categoryNames = ['recent', ...categoryList, 'custom']; const writeableSkinCategories = []; const skinnedCats = []; for (const skin of emojiIndicesByCategoryAndSkin.keys()) { writeableSkinCategories.push(`['${skin}', new Map(${JSON.stringify(Array.from(emojiIndicesByCategoryAndSkin.get(skin)))})]`); skinnedCats.push(`['${skin}', genSkinnedCategories('${skin}')]`); } // Generate contents of emoji.ts out of the emoji.json parsing we did earlier const emojiJSX = `// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. // This file is automatically generated via \`/webapp/channels/build/make_emojis.mjs\`. Do not modify it manually. /* eslint-disable */ import memoize from 'memoize-one'; import emojis from 'utils/emoji.json'; import spriteSheet from '${sheetFile}'; export const Emojis = emojis; export const EmojiIndicesByAlias = new Map(${JSON.stringify(emojiIndicesByAlias)}); export const EmojiIndicesByUnicode = new Map(${JSON.stringify(emojiIndicesByUnicode)}); export const CategoryNames = ${JSON.stringify(categoryNames)}; const AllEmojiIndicesByCategory = new Map(${JSON.stringify(Array.from(emojiIndicesByCategory))}); const EmojiIndicesByCategoryAndSkin = new Map([${writeableSkinCategories.join(', ')}]); const EmojiIndicesByCategoryNoSkin = new Map(${JSON.stringify(Array.from(emojiIndicesByCategoryNoSkin))}); const skinCodes = ${JSON.stringify(skinCodes)}; // Generate the list of indices that belong to each category by an specified skin function genSkinnedCategories(skin: string) { const result = new Map(); for (const cat of CategoryNames) { const indices = []; const skinCat = (EmojiIndicesByCategoryAndSkin.get(skin) || new Map()).get(cat) || []; indices.push(...(EmojiIndicesByCategoryNoSkin.get(cat) || [])); indices.push(...skinCat); result.set(cat, indices); } return result; } const getSkinnedCategories = memoize(genSkinnedCategories); export const EmojiIndicesByCategory = new Map([${skinnedCats.join(', ')}]); `; // Create the emoji.ts file const webappUtilsEmojiTSDir = path.resolve(webappRootDir, 'channels', 'src', 'utils', 'emoji.ts'); endResults.push(writeToFileAndPrint('emoji.ts', webappUtilsEmojiTSDir, emojiJSX)); const serverEmojiDataDir = path.resolve(serverRootDir, 'public', 'model', 'emoji_data.go'); const emojiGo = `// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. // This file is automatically generated via \`/webapp/channels/build/make_emojis.mjs\`. Do not modify it manually. package model var SystemEmojis = map[string]string{ ${emojiImagesByAlias.join(`, `)}, } `; // Create the emoji_data.go file const goPromise = writeToFileAndPrint('emoji_data.go', serverEmojiDataDir, emojiGo); endResults.push(goPromise); // Create individual emoji styles const cssCats = categoryNames.filter((cat) => cat !== 'custom').map((cat) => `.emoji-category-${cat} { background-image: url('${sheetFile}'); }`); const cssEmojis = []; for (const key of emojiFilePositions.keys()) { cssEmojis.push(`.emoji-${key} { background-position: ${emojiFilePositions.get(key)} }`); } const cssRules = `.emojisprite-preview { width: ${EMOJI_SIZE_PADDED}px; max-width: none; height: ${EMOJI_SIZE_PADDED}px; background-repeat: no-repeat; cursor: pointer; -moz-transform: scale(0.5); transform: scale(0.5); transform-origin: 0 0; @supports (zoom: 0.5) { -moz-transform: none; transform: none; zoom: 0.5; } } .emojisprite { width: ${EMOJI_SIZE_PADDED}px; max-width: none; height: ${EMOJI_SIZE_PADDED}px; background-repeat: no-repeat; border-radius: 18px; cursor: pointer; -moz-transform: scale(0.35); transform: scale(0.35); } .emojisprite-loading { width: ${EMOJI_SIZE_PADDED}px; max-width: none; height: ${EMOJI_SIZE_PADDED}px; background-image: none !important; background-repeat: no-repeat; border-radius: 18px; cursor: pointer; -moz-transform: scale(0.35); transform: scale(0.35); } @supports (zoom: 0.35) { .emojisprite, .emojisprite-loading { -moz-transform: none; transform: none; zoom: 0.35; } } ${cssCats.join('\n')} ${cssEmojis.join('\n')} `; // Create the emojisprite.scss file const emojispriteDir = path.resolve(webappRootDir, 'channels', 'src', 'sass', 'components', '_emojisprite.scss'); endResults.push(writeToFileAndPrint('_emojisprite.scss', emojispriteDir, cssRules)); log('', '\n'); log('info', 'Running "make-emojis" script...'); Promise.all(endResults).then(() => { log('warn', 'Remember to npm run "i18n-extract" as categories might have changed.'); log('warn', 'Remember to run "gofmt -w ./server/public/model/emoji_data.go" from the root of the repository.'); log('success', 'make-emojis script completed successfully.'); }).catch((err) => { control.abort(); // cancel any other file writing log('error', `There was an error while running make-emojis script: ${err}`); });