Add emoji autocomplete data and algorithm.

README.md

@@ -28,14 +28,15 @@ The source code is published under GPLv3 with OpenSSL exception, the license is
 * liblzma ([public domain](http://tukaani.org/xz/))
 * Google Breakpad ([License](https://chromium.googlesource.com/breakpad/breakpad/+/master/LICENSE))
 * Google Crashpad ([Apache License 2.0](https://chromium.googlesource.com/crashpad/crashpad/+/master/LICENSE))
-* GYP ([BSD license](https://github.com/bnoordhuis/gyp/blob/master/LICENSE))
+* GYP ([BSD License](https://github.com/bnoordhuis/gyp/blob/master/LICENSE))
 * Ninja ([Apache License 2.0](https://github.com/ninja-build/ninja/blob/master/COPYING))
 * OpenAL Soft ([LGPL](http://kcat.strangesoft.net/openal.html))
-* Opus codec ([BSD license](http://www.opus-codec.org/license/))
+* Opus codec ([BSD License](http://www.opus-codec.org/license/))
 * FFmpeg ([LGPL](https://www.ffmpeg.org/legal.html))
 * Guideline Support Library ([MIT License](https://github.com/Microsoft/GSL/blob/master/LICENSE))
-* Mapbox Variant ([BSD license](https://github.com/mapbox/variant/blob/master/LICENSE))
+* Mapbox Variant ([BSD License](https://github.com/mapbox/variant/blob/master/LICENSE))
 * Open Sans font ([Apache License 2.0](http://www.apache.org/licenses/LICENSE-2.0.html))
+* Emoji alpha codes ([MIT License](https://github.com/emojione/emojione/blob/master/extras/alpha-codes/LICENSE.md))
 
 ## Build instructions
 

Telegram/SourceFiles/chat_helpers/emoji_suggestions.cpp

@@ -0,0 +1,302 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop version of Telegram messaging app, see https://telegram.org
+
+Telegram Desktop is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+It is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+In addition, as a special exception, the copyright holders give permission
+to link the code of portions of this program with the OpenSSL library.
+
+Full license: https://github.com/telegramdesktop/tdesktop/blob/master/LICENSE
+Copyright (c) 2014-2017 John Preston, https://desktop.telegram.org
+*/
+#include "chat_helpers/emoji_suggestions.h"
+
+namespace Ui {
+namespace Emoji {
+namespace {
+
+class Completer {
+public:
+	Completer(const QString &query);
+
+	QVector<Suggestion> resolve();
+
+private:
+	struct Result {
+		gsl::not_null<const Replacement*> replacement;
+		int wordsUsed;
+	};
+
+	static QString NormalizeQuery(const QString &query);
+	void addResult(gsl::not_null<const Replacement*> replacement);
+	bool isDuplicateOfLastResult(gsl::not_null<const Replacement*> replacement) const;
+	bool isBetterThanLastResult(gsl::not_null<const Replacement*> replacement) const;
+	void processInitialList();
+	void filterInitialList();
+	void initWordsTracking();
+	bool matchQueryForCurrentItem();
+	bool matchQueryTailStartingFrom(int position);
+	gsl::span<const QString> findWordsStartingWith(QChar ch);
+	int findEqualCharsCount(int position, const QString *word);
+	QVector<Suggestion> prepareResult();
+
+	std::vector<Result> _result;
+
+	const QString _query;
+	const QChar *_queryBegin = nullptr;
+	int _querySize = 0;
+
+	const std::vector<gsl::not_null<const Replacement*>> *_initialList = nullptr;
+
+	gsl::span<const QString> _currentItemWords;
+	int _currentItemWordsUsedCount = 0;
+
+	class UsedWordGuard {
+	public:
+		UsedWordGuard(QVector<bool> &map, int index);
+		UsedWordGuard(const UsedWordGuard &other) = delete;
+		UsedWordGuard(UsedWordGuard &&other);
+		UsedWordGuard &operator=(const UsedWordGuard &other) = delete;
+		UsedWordGuard &operator=(UsedWordGuard &&other) = delete;
+		explicit operator bool() const;
+		~UsedWordGuard();
+
+	private:
+		QVector<bool> &_map;
+		int _index = 0;
+		bool _guarded = false;
+
+	};
+	QVector<bool> _currentItemWordsUsedMap;
+
+};
+
+Completer::UsedWordGuard::UsedWordGuard(QVector<bool> &map, int index) : _map(map), _index(index) {
+	Expects(_map.size() > _index);
+	if (!_map[_index]) {
+		_guarded = _map[_index] = true;
+	}
+}
+
+Completer::UsedWordGuard::UsedWordGuard(UsedWordGuard &&other) : _map(other._map), _index(other._index), _guarded(base::take(other._guarded)) {
+}
+
+Completer::UsedWordGuard::operator bool() const {
+	return _guarded;
+}
+
+Completer::UsedWordGuard::~UsedWordGuard() {
+	if (_guarded) {
+		_map[_index] = false;
+	}
+}
+
+Completer::Completer(const QString &query) : _query(NormalizeQuery(query)) {
+}
+
+// Remove all non-letters-or-numbers.
+// Leave '-' and '+' only if they're followed by a number or
+// at the end of the query (so it is possibly followed by a number).
+QString Completer::NormalizeQuery(const QString &query) {
+	auto result = query;
+	auto copyFrom = query.constData();
+	auto e = copyFrom + query.size();
+	auto copyTo = (QChar*)nullptr;
+	for (auto i = query.constData(); i != e; ++i) {
+		if (i->isLetterOrNumber()) {
+			continue;
+		} else if (*i == '-' || *i == '+') {
+			if (i + 1 == e || (i + 1)->isNumber()) {
+				continue;
+			}
+		}
+		if (i > copyFrom) {
+			if (!copyTo) copyTo = result.data();
+			memcpy(copyTo, copyFrom, (i - copyFrom) * sizeof(QChar));
+			copyTo += (i - copyFrom);
+		}
+		copyFrom = i + 1;
+	}
+	if (copyFrom == query.constData()) {
+		return query;
+	} else if (e > copyFrom) {
+		if (!copyTo) copyTo = result.data();
+		memcpy(copyTo, copyFrom, (e - copyFrom) * sizeof(QChar));
+		copyTo += (e - copyFrom);
+	}
+	result.chop(result.constData() + result.size() - copyTo);
+	return result;
+}
+
+QVector<Suggestion> Completer::resolve() {
+	_queryBegin = _query.constData();
+	_querySize = _query.size();
+	if (!_querySize) {
+		return QVector<Suggestion>();
+	}
+	_initialList = Ui::Emoji::GetReplacements(*_queryBegin);
+	if (!_initialList) {
+		return QVector<Suggestion>();
+	}
+	_result.reserve(_initialList->size());
+	processInitialList();
+	return prepareResult();
+}
+
+bool Completer::isDuplicateOfLastResult(gsl::not_null<const Replacement*> item) const {
+	if (_result.empty()) {
+		return false;
+	}
+	return (_result.back().replacement->id == item->id);
+}
+
+bool Completer::isBetterThanLastResult(gsl::not_null<const Replacement*> item) const {
+	Expects(!_result.empty());
+	auto &last = _result.back();
+	if (_currentItemWordsUsedCount < last.wordsUsed) {
+		return true;
+	}
+
+	auto firstCharOfQuery = _query[0];
+	auto firstCharAfterColonLast = last.replacement->replacement[1];
+	auto firstCharAfterColonCurrent = item->replacement[1];
+	auto goodLast = (firstCharAfterColonLast == firstCharOfQuery);
+	auto goodCurrent = (firstCharAfterColonCurrent == firstCharOfQuery);
+	return !goodLast && goodCurrent;
+}
+
+void Completer::addResult(gsl::not_null<const Replacement*> item) {
+	if (!isDuplicateOfLastResult(item)) {
+		_result.push_back({ item, _currentItemWordsUsedCount });
+	} else if (isBetterThanLastResult(item)) {
+		_result.back() = { item, _currentItemWordsUsedCount };
+	}
+}
+
+void Completer::processInitialList() {
+	if (_querySize > 1) {
+		filterInitialList();
+	} else {
+		_currentItemWordsUsedCount = 1;
+		for (auto item : *_initialList) {
+			addResult(item);
+		}
+	}
+}
+
+void Completer::initWordsTracking() {
+	auto maxWordsCount = 0;
+	for (auto item : *_initialList) {
+		accumulate_max(maxWordsCount, item->words.size());
+	}
+	_currentItemWordsUsedMap = QVector<bool>(maxWordsCount, false);
+}
+
+void Completer::filterInitialList() {
+	initWordsTracking();
+	for (auto item : *_initialList) {
+		_currentItemWords = gsl::make_span(item->words);
+		_currentItemWordsUsedCount = 1;
+		if (matchQueryForCurrentItem()) {
+			addResult(item);
+		}
+		_currentItemWordsUsedCount = 0;
+	}
+}
+
+bool Completer::matchQueryForCurrentItem() {
+	Expects(!_currentItemWords.empty());
+	if (_currentItemWords.size() < 2) {
+		return _currentItemWords.data()->startsWith(_query);
+	}
+	return matchQueryTailStartingFrom(0);
+}
+
+bool Completer::matchQueryTailStartingFrom(int position) {
+	auto charsLeftToMatch = (_querySize - position);
+	if (!charsLeftToMatch) {
+		return true;
+	}
+
+	auto firstCharToMatch = *(_queryBegin + position);
+	auto foundWords = findWordsStartingWith(firstCharToMatch);
+
+	for (auto word = foundWords.data(), foundWordsEnd = word + foundWords.size(); word != foundWordsEnd; ++word) {
+		auto wordIndex = word - _currentItemWords.data();
+		if (auto guard = UsedWordGuard(_currentItemWordsUsedMap, wordIndex)) {
+			++_currentItemWordsUsedCount;
+			auto equalCharsCount = findEqualCharsCount(position, word);
+			for (auto check = equalCharsCount; check != 0; --check) {
+				if (matchQueryTailStartingFrom(position + check)) {
+					return true;
+				}
+			}
+			--_currentItemWordsUsedCount;
+		}
+	}
+	return false;
+}
+
+int Completer::findEqualCharsCount(int position, const QString *word) {
+	auto charsLeft = (_querySize - position);
+	auto wordBegin = word->constData();
+	auto wordSize = word->size();
+	auto possibleEqualCharsCount = qMin(charsLeft, wordSize);
+	for (auto equalTill = 1; equalTill != possibleEqualCharsCount; ++equalTill) {
+		auto wordCh = *(wordBegin + equalTill);
+		auto queryCh = *(_queryBegin + position + equalTill);
+		if (wordCh != queryCh) {
+			return equalTill;
+		}
+	}
+	return possibleEqualCharsCount;
+}
+
+QVector<Suggestion> Completer::prepareResult() {
+	auto firstCharOfQuery = _query[0];
+	std::stable_partition(_result.begin(), _result.end(), [firstCharOfQuery](Result &result) {
+		auto firstCharAfterColon = result.replacement->replacement[1];
+		return (firstCharAfterColon == firstCharOfQuery);
+	});
+	std::stable_partition(_result.begin(), _result.end(), [](Result &result) {
+		return (result.wordsUsed < 2);
+	});
+	std::stable_partition(_result.begin(), _result.end(), [](Result &result) {
+		return (result.wordsUsed < 3);
+	});
+
+	auto result = QVector<Suggestion>();
+	result.reserve(_result.size());
+	for (auto &item : _result) {
+		result.push_back({ item.replacement->id, item.replacement->label, item.replacement->replacement });
+	}
+	return result;
+}
+
+gsl::span<const QString> Completer::findWordsStartingWith(QChar ch) {
+	auto begin = std::lower_bound(_currentItemWords.cbegin(), _currentItemWords.cend(), ch, [](const QString &word, QChar ch) {
+		return word[0] < ch;
+	});
+	auto end = std::upper_bound(_currentItemWords.cbegin(), _currentItemWords.cend(), ch, [](QChar ch, const QString &word) {
+		return ch < word[0];
+	});
+	return _currentItemWords.subspan(begin - _currentItemWords.cbegin(), end - begin);
+}
+
+} // namespace
+
+QVector<Suggestion> GetSuggestions(const QString &query) {
+	return Completer(query).resolve();
+}
+
+} // namespace Emoji
+} // namespace Ui

Telegram/SourceFiles/chat_helpers/emoji_suggestions.h

@@ -0,0 +1,35 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop version of Telegram messaging app, see https://telegram.org
+
+Telegram Desktop is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+It is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+In addition, as a special exception, the copyright holders give permission
+to link the code of portions of this program with the OpenSSL library.
+
+Full license: https://github.com/telegramdesktop/tdesktop/blob/master/LICENSE
+Copyright (c) 2014-2017 John Preston, https://desktop.telegram.org
+*/
+#pragma once
+
+namespace Ui {
+namespace Emoji {
+
+struct Suggestion {
+	QString id;
+	QString label;
+	QString replacement;
+};
+
+QVector<Suggestion> GetSuggestions(const QString &query);
+
+} // namespace Emoji
+} // namespace Ui

Telegram/SourceFiles/chat_helpers/emoji_suggestions_widget.cpp

@@ -0,0 +1,22 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop version of Telegram messaging app, see https://telegram.org
+
+Telegram Desktop is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+It is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+In addition, as a special exception, the copyright holders give permission
+to link the code of portions of this program with the OpenSSL library.
+
+Full license: https://github.com/telegramdesktop/tdesktop/blob/master/LICENSE
+Copyright (c) 2014-2017 John Preston, https://desktop.telegram.org
+*/
+#include "chat_helpers/emoji_suggestions_widget.h"
+

Telegram/SourceFiles/chat_helpers/emoji_suggestions_widget.h

@@ -0,0 +1,22 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop version of Telegram messaging app, see https://telegram.org
+
+Telegram Desktop is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+It is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+In addition, as a special exception, the copyright holders give permission
+to link the code of portions of this program with the OpenSSL library.
+
+Full license: https://github.com/telegramdesktop/tdesktop/blob/master/LICENSE
+Copyright (c) 2014-2017 John Preston, https://desktop.telegram.org
+*/
+#pragma once
+

Telegram/SourceFiles/codegen/emoji/data.cpp

@@ -1835,6 +1835,34 @@ void fillReplaces(Data &result) {
 	}
 }
 
+bool AddItemBeforeItem(const InputId &add, const InputId &before) {
+	auto addToCategory = (InputCategory*)nullptr;
+	auto addBeforeIterator = InputCategory::iterator();
+	for (auto category : {
+		&Category1,
+		&Category2,
+		&Category3,
+		&Category4,
+		&Category5,
+		&Category6,
+		&Category7,
+	}) {
+		for (auto i = category->begin(), e = category->end(); i != e; ++i) {
+			if (*i == add) {
+				return true;
+			} else if (*i == before) {
+				addToCategory = category;
+				addBeforeIterator = i;
+			}
+		}
+	}
+	if (!addToCategory) {
+		return false;
+	}
+	addToCategory->insert(addBeforeIterator, add);
+	return true;
+}
+
 } // namespace
 
 common::LogStream logDataError() {
@@ -1849,7 +1877,12 @@ Data PrepareData() {
 		return Data();
 	}
 
-	vector<const InputCategory*> categories = {
+	// Manually add :speech_left: emoji before eye-with-speech emoji.
+	if (!AddItemBeforeItem({ 0xD83DDDE8U }, { 0xD83DDC41U, 0x200DU, 0xD83DDDE8U })) {
+		return Data();
+	}
+
+	for (auto category : {
 		&Category1,
 		&Category2,
 		&Category3,
@@ -1857,8 +1890,7 @@ Data PrepareData() {
 		&Category5,
 		&Category6,
 		&Category7,
-	};
+	}) {
-	for (auto category : categories) {
 		appendCategory(result, *category, variatedIds);
 		if (result.list.empty()) {
 			return Data();

Telegram/SourceFiles/codegen/emoji/generator.cpp

@@ -43,7 +43,12 @@ namespace codegen {
 namespace emoji {
 namespace {
 
-constexpr int kErrorCantWritePath = 851;
+constexpr auto kErrorCantWritePath = 851;
+
+constexpr auto kOriginalBits = 12;
+constexpr auto kIdSizeBits = 6;
+constexpr auto kColumnBits = 6;
+constexpr auto kRowBits = 6;
 
 common::ProjectInfo Project = {
 	"codegen_emoji",
@@ -124,19 +129,23 @@ Generator::Generator(const Options &options) : project_(Project)
 #ifdef SUPPORT_IMAGE_GENERATION
 , writeImages_(options.writeImages)
 #endif // SUPPORT_IMAGE_GENERATION
-, data_(PrepareData()) {
+, data_(PrepareData())
+, replaces_(PrepareReplaces(options.replacesPath)) {
 	QDir dir(options.outputPath);
 	if (!dir.mkpath(".")) {
 		common::logError(kErrorCantWritePath, "Command Line") << "can not open path for writing: " << dir.absolutePath().toStdString();
 		data_ = Data();
 	}
+	if (!CheckAndConvertReplaces(replaces_, data_)) {
+		replaces_ = Replaces(replaces_.filename);
+	}
 
 	outputPath_ = dir.absolutePath() + "/emoji";
 	spritePath_ = dir.absolutePath() + "/emoji";
 }
 
 int Generator::generate() {
-	if (data_.list.empty()) {
+	if (data_.list.empty() || replaces_.list.isEmpty()) {
 		return -1;
 	}
 
@@ -274,6 +283,9 @@ std::vector<One> Items;\n\
 	if (!writeSections()) {
 		return false;
 	}
+	if (!writeReplacements()) {
+		return false;
+	}
 	if (!writeFindReplace()) {
 		return false;
 	}
@@ -299,11 +311,17 @@ EmojiPtr Find(const QChar *start, const QChar *end, int *outLength) {\n\
 \n\
 void Init() {\n\
 	auto id = IdData;\n\
+	auto takeString = [&id](int size) {\n\
+		auto result = QString::fromRawData(reinterpret_cast<const QChar*>(id), size);\n\
+		id += size;\n\
+		return result;\n\
+	};\n\
+\n\
 	Items.reserve(base::array_size(Data));\n\
 	for (auto &data : Data) {\n\
-		Items.emplace_back(QString::fromRawData(id, data.idSize), data.column, data.row, data.postfixed, data.variated, data.original ? &Items[data.original - 1] : nullptr, One::CreationTag());\n\
+		Items.emplace_back(takeString(data.idSize), uint16(data.column), uint16(data.row), bool(data.postfixed), bool(data.variated), data.original ? &Items[data.original - 1] : nullptr, One::CreationTag());\n\
-		id += data.idSize;\n\
 	}\n\
+	InitReplacements();\n\
 }\n\
 \n";
 	source_->popNamespace();
@@ -311,6 +329,9 @@ void Init() {\n\
 	if (!writeGetSections()) {
 		return false;
 	}
+	if (!writeGetReplacements()) {
+		return false;
+	}
 
 	return source_->finalize();
 }
@@ -359,6 +380,15 @@ int Index();\n\
 \n\
 int GetSectionCount(Section section);\n\
 EmojiPack GetSection(Section section);\n\
+\n\
+struct Replacement {\n\
+	QString id;\n\
+	QString replacement;\n\
+	QString label;\n\
+	QVector<QString> words;\n\
+};\n\
+\n\
+const std::vector<gsl::not_null<const Replacement*>> *GetReplacements(QChar first);\n\
 \n";
 	return header->finalize();
 }
@@ -371,7 +401,9 @@ bool Generator::enumerateWholeList(Callback callback) {
 	auto variated = -1;
 	auto coloredCount = 0;
 	for (auto &item : data_.list) {
-		callback(item.id, column, row, item.postfixed, item.variated, item.colored, variated);
+		if (!callback(item.id, column, row, item.postfixed, item.variated, item.colored, variated)) {
+			return false;
+		}
 		if (coloredCount > 0 && (item.variated || !item.colored)) {
 			if (!colorsCount_) {
 				colorsCount_ = coloredCount;
@@ -404,44 +436,44 @@ bool Generator::enumerateWholeList(Callback callback) {
 bool Generator::writeInitCode() {
 	source_->stream() << "\
 struct DataStruct {\n\
-	ushort idSize;\n\
+	ushort original : " << kOriginalBits << ";\n\
-	ushort column;\n\
+	uchar idSize : " << kIdSizeBits << ";\n\
-	ushort row;\n\
+	uchar column : " << kColumnBits << ";\n\
-	ushort original;\n\
+	uchar row : " << kRowBits << ";\n\
-	bool postfixed;\n\
+	bool postfixed : 1;\n\
-	bool variated;\n\
+	bool variated : 1;\n\
 };\n\
 \n\
-QChar IdData[] = {";
+const ushort IdData[] = {";
-	auto count = 0;
+	startBinary();
-	auto fulllength = 0;
+	if (!enumerateWholeList([this](Id id, int column, int row, bool isPostfixed, bool isVariated, bool isColored, int original) {
-	if (!enumerateWholeList([this, &count, &fulllength](Id id, int column, int row, bool isPostfixed, bool isVariated, bool isColored, int original) {
+		return writeStringBinary(id);
-		for (auto ch : id) {
-			if (fulllength > 0) source_->stream() << ",";
-			if (!count++) {
-				source_->stream() << "\n";
-			} else {
-				if (count == 12) {
-					count = 0;
-				}
-				source_->stream() << " ";
-			}
-			source_->stream() << "0x" << QString::number(ch.unicode(), 16);
-			++fulllength;
-		}
 	})) {
 		return false;
 	}
-	if (fulllength >= std::numeric_limits<ushort>::max()) {
+	if (_binaryFullLength >= std::numeric_limits<ushort>::max()) {
 		logDataError() << "Too many IdData elements.";
 		return false;
 	}
 	source_->stream() << " };\n\
 \n\
-DataStruct Data[] = {\n";
+const DataStruct Data[] = {\n";
 	if (!enumerateWholeList([this](Id id, int column, int row, bool isPostfixed, bool isVariated, bool isColored, int original) {
+		if (original + 1 >= (1 << kOriginalBits)) {
+			logDataError() << "Too many entries.";
+			return false;
+		}
+		if (id.size() >= (1 << kIdSizeBits)) {
+			logDataError() << "Too large id.";
+			return false;
+		}
+		if (column >= (1 << kColumnBits) || row >= (1 << kRowBits)) {
+			logDataError() << "Bad row-column.";
+			return false;
+		}
 		source_->stream() << "\
-	{ ushort(" << id.size() << "), ushort(" << column << "), ushort(" << row << "), ushort(" << (isColored ? (original + 1) : 0) << "), " << (isPostfixed ? "true" : "false") << ", " << (isVariated ? "true" : "false") << " },\n";
+	{ ushort(" << (isColored ? (original + 1) : 0) << "), uchar(" << id.size() << "), uchar(" << column << "), uchar(" << row << "), " << (isPostfixed ? "true" : "false") << ", " << (isVariated ? "true" : "false") << " },\n";
+		return true;
 	})) {
 		return false;
 	}
@@ -454,26 +486,16 @@ DataStruct Data[] = {\n";
 
 bool Generator::writeSections() {
 	source_->stream() << "\
-ushort SectionData[] = {";
+const ushort SectionData[] = {";
-	auto count = 0, fulllength = 0;
+	startBinary();
 	for (auto &category : data_.categories) {
 		for (auto index : category) {
-			if (fulllength > 0) source_->stream() << ",";
+			writeIntBinary(index);
-			if (!count++) {
-				source_->stream() << "\n";
-			} else {
-				if (count == 12) {
-					count = 0;
-				}
-				source_->stream() << " ";
-			}
-			source_->stream() << index;
-			++fulllength;
 		}
 	}
 	source_->stream() << " };\n\
 \n\
-EmojiPack fillSection(int offset, int size) {\n\
+EmojiPack FillSection(int offset, int size) {\n\
 	auto result = EmojiPack();\n\
 	result.reserve(size);\n\
 	for (auto index : gsl::make_span(SectionData + offset, size)) {\n\
@@ -484,6 +506,115 @@ EmojiPack fillSection(int offset, int size) {\n\
 	return true;
 }
 
+bool Generator::writeReplacements() {
+	QMap<QChar, QVector<int>> byCharIndices;
+	source_->stream() << "\
+struct ReplacementStruct {\n\
+	uchar idSize;\n\
+	uchar replacementSize;\n\
+	uchar wordsCount;\n\
+};\n\
+\n\
+const ushort ReplacementData[] = {";
+	startBinary();
+	for (auto i = 0, size = replaces_.list.size(); i != size; ++i) {
+		auto &replace = replaces_.list[i];
+		if (!writeStringBinary(replace.id)) {
+			return false;
+		}
+		if (!writeStringBinary(replace.replacement)) {
+			return false;
+		}
+		for (auto &word : replace.words) {
+			if (!writeStringBinary(word)) {
+				return false;
+			}
+			auto &index = byCharIndices[word[0]];
+			if (index.isEmpty() || index.back() != i) {
+				index.push_back(i);
+			}
+		}
+	}
+	source_->stream() << " };\n\
+\n\
+const uchar ReplacementWordLengths[] = {";
+	startBinary();
+	for (auto &replace : replaces_.list) {
+		auto wordLengths = QStringList();
+		for (auto &word : replace.words) {
+			writeIntBinary(word.size());
+		}
+	}
+	source_->stream() << " };\n\
+\n\
+const ReplacementStruct ReplacementInitData[] = {\n";
+	for (auto &replace : replaces_.list) {
+		source_->stream() << "\
+	{ uchar(" << replace.id.size() << "), uchar(" << replace.replacement.size() << "), uchar(" << replace.words.size() << ") },\n";
+	}
+	source_->stream() << "};\n\
+\n\
+const ushort ReplacementIndices[] = {";
+	startBinary();
+	for (auto &byCharIndex : byCharIndices) {
+		for (auto index : byCharIndex) {
+			writeIntBinary(index);
+		}
+	}
+	source_->stream() << " };\n\
+\n\
+struct ReplacementIndexStruct {\n\
+	ushort ch;\n\
+	ushort count;\n\
+};\n\
+\n\
+const ReplacementIndexStruct ReplacementIndexData[] = {\n";
+	startBinary();
+	for (auto i = byCharIndices.cbegin(), e = byCharIndices.cend(); i != e; ++i) {
+		source_->stream() << "\
+	{ ushort(" << i.key().unicode() << "), ushort(" << i.value().size() << ") },\n";
+	}
+	source_->stream() << "};\n\
+\n\
+std::vector<Replacement> Replacements;\n\
+std::map<ushort, std::vector<gsl::not_null<const Replacement*>>> ReplacementsMap;\n\
+\n\
+void InitReplacements() {\n\
+	auto data = ReplacementData;\n\
+	auto takeString = [&data](int size) {\n\
+		auto result = QString::fromRawData(reinterpret_cast<const QChar*>(data), size);\n\
+		data += size;\n\
+		return result;\n\
+	};\n\
+	auto wordSize = ReplacementWordLengths;\n\
+\n\
+	Replacements.reserve(base::array_size(ReplacementInitData));\n\
+	for (auto item : ReplacementInitData) {\n\
+		auto id = takeString(item.idSize);\n\
+		auto replacement = takeString(item.replacementSize);\n\
+		auto label = replacement;\n\
+		auto words = QVector<QString>();\n\
+		words.reserve(item.wordsCount);\n\
+		for (auto i = 0; i != item.wordsCount; ++i) {\n\
+			words.push_back(takeString(*wordSize++));\n\
+		}\n\
+		Replacements.push_back({ std::move(id), std::move(replacement), std::move(label), std::move(words) });\n\
+	}\n\
+\n\
+	auto indices = ReplacementIndices;\n\
+	auto items = &Replacements[0];\n\
+	for (auto item : ReplacementIndexData) {\n\
+		auto index = std::vector<gsl::not_null<const Replacement*>>();\n\
+		index.reserve(item.count);\n\
+		for (auto i = 0; i != item.count; ++i) {\n\
+			index.push_back(items + (*indices++));\n\
+		}\n\
+		ReplacementsMap.emplace(item.ch, std::move(index));\n\
+	}\n\
+}\n";
+	return true;
+}
+
 bool Generator::writeGetSections() {
 	constexpr const char *sectionNames[] = {
 		"Section::People",
@@ -534,7 +665,7 @@ EmojiPack GetSection(Section section) {\n\
 		source_->stream() << "\
 \n\
 	case " << name << ": {\n\
-		static auto result = fillSection(" << offset << ", " << category.size() << ");\n\
+		static auto result = FillSection(" << offset << ", " << category.size() << ");\n\
 		return result;\n\
 	} break;\n";
 		offset += category.size();
@@ -702,5 +833,54 @@ bool Generator::writeFindFromDictionary(const std::map<QString, int, std::greate
 	return true;
 }
 
+bool Generator::writeGetReplacements() {
+	source_->stream() << "\
+const std::vector<gsl::not_null<const Replacement*>> *GetReplacements(QChar first) {\n\
+	auto it = ReplacementsMap.find(first.unicode());\n\
+	return (it == ReplacementsMap.cend()) ? nullptr : &it->second;\n\
+}\n\
+\n";
+	return true;
+}
+
+void Generator::startBinary() {
+	_binaryFullLength = _binaryCount = 0;
+}
+
+bool Generator::writeStringBinary(const QString &string) {
+	if (string.size() >= 256) {
+		logDataError() << "Too long string: " << string.toStdString();
+		return false;
+	}
+	for (auto ch : string) {
+		if (_binaryFullLength > 0) source_->stream() << ",";
+		if (!_binaryCount++) {
+			source_->stream() << "\n";
+		} else {
+			if (_binaryCount == 12) {
+				_binaryCount = 0;
+			}
+			source_->stream() << " ";
+		}
+		source_->stream() << "0x" << QString::number(ch.unicode(), 16);
+		++_binaryFullLength;
+	}
+	return true;
+}
+
+void Generator::writeIntBinary(int data) {
+	if (_binaryFullLength > 0) source_->stream() << ",";
+	if (!_binaryCount++) {
+		source_->stream() << "\n";
+	} else {
+		if (_binaryCount == 12) {
+			_binaryCount = 0;
+		}
+		source_->stream() << " ";
+	}
+	source_->stream() << data;
+	++_binaryFullLength;
+}
+
 } // namespace emoji
 } // namespace codegen

Telegram/SourceFiles/codegen/emoji/generator.h

@@ -26,6 +26,7 @@ Copyright (c) 2014-2017 John Preston, https://desktop.telegram.org
 #include "codegen/common/cpp_file.h"
 #include "codegen/emoji/options.h"
 #include "codegen/emoji/data.h"
+#include "codegen/emoji/replaces.h"
 
 namespace codegen {
 namespace emoji {
@@ -52,10 +53,15 @@ private:
 
 	bool writeInitCode();
 	bool writeSections();
+	bool writeReplacements();
 	bool writeGetSections();
 	bool writeFindReplace();
 	bool writeFind();
 	bool writeFindFromDictionary(const std::map<QString, int, std::greater<QString>> &dictionary, bool skipPostfixes = false);
+	bool writeGetReplacements();
+	void startBinary();
+	bool writeStringBinary(const QString &string);
+	void writeIntBinary(int data);
 
 	const common::ProjectInfo &project_;
 	int colorsCount_ = 0;
@@ -66,6 +72,10 @@ private:
 	QString spritePath_;
 	std::unique_ptr<common::CppFile> source_;
 	Data data_;
+	Replaces replaces_;
+
+	int _binaryFullLength = 0;
+	int _binaryCount = 0;
 
 };
 

Telegram/SourceFiles/codegen/emoji/options.cpp

@@ -29,6 +29,8 @@ namespace emoji {
 namespace {
 
 constexpr int kErrorOutputPathExpected      = 902;
+constexpr int kErrorReplacesPathExpected    = 903;
+constexpr int kErrorOneReplacesPathExpected = 904;
 
 } // namespace
 
@@ -54,11 +56,19 @@ Options parseOptions() {
 		} else if (arg == "--images") {
 			result.writeImages = true;
 #endif // SUPPORT_IMAGE_GENERATION
+		} else if (result.replacesPath.isEmpty()) {
+			result.replacesPath = arg;
+		} else {
+			logError(kErrorOneReplacesPathExpected, "Command Line") << "only one replaces path expected";
+			return Options();
 		}
 	}
 	if (result.outputPath.isEmpty()) {
 		logError(kErrorOutputPathExpected, "Command Line") << "output path expected";
 		return Options();
+	} else if (result.replacesPath.isEmpty()) {
+		logError(kErrorReplacesPathExpected, "Command Line") << "replaces path expected";
+		return Options();
 	}
 	return result;
 }

Telegram/SourceFiles/codegen/emoji/options.h

@@ -28,7 +28,7 @@ namespace emoji {
 
 struct Options {
 	QString outputPath = ".";
-
+	QString replacesPath;
 #ifdef SUPPORT_IMAGE_GENERATION
 	bool writeImages = false;
 #endif // SUPPORT_IMAGE_GENERATION

Telegram/SourceFiles/codegen/emoji/replaces.cpp

@@ -0,0 +1,484 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop version of Telegram messaging app, see https://telegram.org
+
+Telegram Desktop is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+It is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+In addition, as a special exception, the copyright holders give permission
+to link the code of portions of this program with the OpenSSL library.
+
+Full license: https://github.com/telegramdesktop/tdesktop/blob/master/LICENSE
+Copyright (c) 2014-2017 John Preston, https://desktop.telegram.org
+*/
+#include "codegen/emoji/replaces.h"
+
+#include "codegen/emoji/data.h"
+#include <QtCore/QFile>
+#include <QtCore/QJsonDocument>
+#include <QtCore/QJsonObject>
+#include <QtCore/QRegularExpression>
+
+namespace codegen {
+namespace emoji {
+namespace {
+
+constexpr auto kErrorBadReplaces = 402;
+
+common::LogStream logReplacesError(const QString &filename) {
+	return common::logError(kErrorBadReplaces, filename) << "Bad data: ";
+}
+
+auto RegExpCode = QRegularExpression("^:[\\+\\-a-z0-9_]+:$");
+auto RegExpTone = QRegularExpression("_tone[0-9]");
+auto RegExpHex = QRegularExpression("^[0-9a-f]+$");
+
+class ReplacementWords {
+public:
+	ReplacementWords(const QString &string);
+	QVector<QString> result() const;
+
+private:
+	friend ReplacementWords operator+(const ReplacementWords &a, const ReplacementWords &b);
+
+	QMap<QString, int> wordsWithCounts_;
+
+};
+
+ReplacementWords::ReplacementWords(const QString &string) {
+	auto feedWord = [this](QString &word) {
+		if (!word.isEmpty()) {
+			++wordsWithCounts_[word];
+			word.clear();
+		}
+	};
+	// Split by all non-letters-or-numbers.
+	// Leave '-' and '+' inside a word only if they're followed by a number.
+	auto word = QString();
+	for (auto i = string.cbegin(), e = string.cend(); i != e; ++i) {
+		if (i->isLetterOrNumber()) {
+			word.append(*i);
+			continue;
+		} else if (*i == '-' || *i == '+') {
+			if (i + 1 != e && (i + 1)->isNumber()) {
+				word.append(*i);
+				continue;
+			}
+		}
+		feedWord(word);
+	}
+	feedWord(word);
+}
+
+QVector<QString> ReplacementWords::result() const {
+	auto result = QVector<QString>();
+	for (auto i = wordsWithCounts_.cbegin(), e = wordsWithCounts_.cend(); i != e; ++i) {
+		for (auto j = 0, count = i.value(); j != count; ++j) {
+			result.push_back(i.key());
+		}
+	}
+	return result;
+}
+
+ReplacementWords operator+(const ReplacementWords &a, const ReplacementWords &b) {
+	ReplacementWords result = a;
+	for (auto i = b.wordsWithCounts_.cbegin(), e = b.wordsWithCounts_.cend(); i != e; ++i) {
+		auto j = result.wordsWithCounts_.constFind(i.key());
+		if (j == result.wordsWithCounts_.cend() || j.value() < i.value()) {
+			result.wordsWithCounts_[i.key()] = i.value();
+		}
+	}
+	return result;
+}
+
+bool AddReplacement(Replaces &result, const Id &id, const QString &replacement, const QString &name) {
+	auto replace = Replace();
+	replace.id = id;
+	replace.replacement = replacement;
+	replace.words = (ReplacementWords(replacement)).result();// + ReplacementWords(name)).result();
+	if (replace.words.isEmpty()) {
+		logReplacesError(result.filename) << "Child '" << replacement.toStdString() << "' has no words.";
+		return false;
+	}
+	result.list.push_back(replace);
+	return true;
+}
+
+QString ComposeString(const std::initializer_list<QChar> &chars) {
+	auto result = QString();
+	result.reserve(chars.size());
+	for (auto ch : chars) {
+		result.append(ch);
+	}
+	return result;
+}
+
+const auto NotSupported = ([] {
+	auto result = QSet<QString>();
+	auto insert = [&result](auto... args) {
+		result.insert(ComposeString({ args... }));
+	};
+	insert(0x0023, 0xFE0F); // :pound_symbol:
+	insert(0x002A, 0xFE0F); // :asterisk_symbol:
+	for (auto i = 0; i != 10; ++i) {
+		insert(0x0030 + i, 0xFE0F); // :digit_zero: ... :digit_nine:
+	}
+	for (auto i = 0; i != 5; ++i) {
+		insert(0xD83C, 0xDFFB + i); // :tone1: ... :tone5:
+	}
+	for (auto i = 0; i != 26; ++i) {
+		insert(0xD83C, 0xDDE6 + i); // :regional_indicator_a: ... :regional_indicator_z:
+	}
+	insert(0xD83C, 0xDDFA, 0xD83C, 0xDDF3); // :united_nations:
+
+	insert(0xD83C, 0xDFF4, 0xDB40, 0xDC67, 0xDB40, 0xDC62, 0xDB40, 0xDC65, 0xDB40, 0xDC6E, 0xDB40, 0xDC67, 0xDB40, 0xDC7F); // :england:
+	insert(0xD83C, 0xDFF4, 0xDB40, 0xDC67, 0xDB40, 0xDC62, 0xDB40, 0xDC73, 0xDB40, 0xDC63, 0xDB40, 0xDC74, 0xDB40, 0xDC7F); // :scotland:
+	insert(0xD83C, 0xDFF4, 0xDB40, 0xDC67, 0xDB40, 0xDC62, 0xDB40, 0xDC77, 0xDB40, 0xDC6C, 0xDB40, 0xDC73, 0xDB40, 0xDC7F); // :wales:
+
+	insert(0xD83D, 0xDEF7); // :sled:
+	insert(0xD83D, 0xDEF8); // :flying_saucer:
+	insert(0xD83E, 0xDD1F); // :love_you_gesture:
+	insert(0xD83E, 0xDD28); // :face_with_raised_eyebrow:
+	insert(0xD83E, 0xDD29); // :star_struck:
+	insert(0xD83E, 0xDD2A); // :crazy_face:
+	insert(0xD83E, 0xDD2B); // :shushing_face:
+	insert(0xD83E, 0xDD2C); // :face_with_symbols_over_mouth:
+	insert(0xD83E, 0xDD2D); // :face_with_hand_over_mouth:
+	insert(0xD83E, 0xDD2E); // :face_vomiting:
+	insert(0xD83E, 0xDD2F); // :exploding_head:
+	insert(0xD83E, 0xDD31); // :breast_feeding:
+	insert(0xD83E, 0xDD32); // :palms_up_together:
+	insert(0xD83E, 0xDD4C); // :curling_stone:
+	insert(0xD83E, 0xDD5F); // :dumpling:
+	insert(0xD83E, 0xDD60); // :fortune_cookie:
+	insert(0xD83E, 0xDD61); // :takeout_box:
+	insert(0xD83E, 0xDD62); // :chopsticks:
+	insert(0xD83E, 0xDD63); // :bowl_with_spoon:
+	insert(0xD83E, 0xDD64); // :cup_with_straw:
+	insert(0xD83E, 0xDD65); // :coconut:
+	insert(0xD83E, 0xDD66); // :broccoli:
+	insert(0xD83E, 0xDD67); // :pie:
+	insert(0xD83E, 0xDD68); // :pretzel:
+	insert(0xD83E, 0xDD69); // :cut_of_meat:
+	insert(0xD83E, 0xDD6A); // :sandwich:
+	insert(0xD83E, 0xDD6B); // :canned_food:
+	insert(0xD83E, 0xDD92); // :giraffe:
+	insert(0xD83E, 0xDD93); // :zebra:
+	insert(0xD83E, 0xDD94); // :hedgehog:
+	insert(0xD83E, 0xDD95); // :sauropod:
+	insert(0xD83E, 0xDD96); // :t_rex:
+	insert(0xD83E, 0xDD97); // :cricket:
+	insert(0xD83E, 0xDDD0); // :face_with_monocle:
+	insert(0xD83E, 0xDDD1); // :adult:
+	insert(0xD83E, 0xDDD2); // :child:
+	insert(0xD83E, 0xDDD3); // :older_adult:
+	insert(0xD83E, 0xDDD4); // :bearded_person:
+	insert(0xD83E, 0xDDD5); // :woman_with_headscarf:
+	insert(0xD83E, 0xDDD6); // :person_in_steamy_room:
+	insert(0xD83E, 0xDDD6, 0x200D, 0x2640, 0xFE0F); // :woman_in_steamy_room:
+	insert(0xD83E, 0xDDD6, 0x200D, 0x2642, 0xFE0F); // :man_in_steamy_room:
+	insert(0xD83E, 0xDDD7); // :person_climbing:
+	insert(0xD83E, 0xDDD7, 0x200D, 0x2640, 0xFE0F); // :woman_climbing:
+	insert(0xD83E, 0xDDD7, 0x200D, 0x2642, 0xFE0F); // :man_climbing:
+	insert(0xD83E, 0xDDD8); // :person_in_lotus_position:
+	insert(0xD83E, 0xDDD8, 0x200D, 0x2640, 0xFE0F); // :woman_in_lotus_position:
+	insert(0xD83E, 0xDDD8, 0x200D, 0x2642, 0xFE0F); // :man_in_lotus_position:
+	insert(0xD83E, 0xDDD9); // :mage:
+	insert(0xD83E, 0xDDD9, 0x200D, 0x2640, 0xFE0F); // :woman_mage:
+	insert(0xD83E, 0xDDD9, 0x200D, 0x2642, 0xFE0F); // :man_mage:
+	insert(0xD83E, 0xDDDA); // :fairy:
+	insert(0xD83E, 0xDDDA, 0x200D, 0x2640, 0xFE0F); // :woman_fairy:
+	insert(0xD83E, 0xDDDA, 0x200D, 0x2642, 0xFE0F); // :man_fairy:
+	insert(0xD83E, 0xDDDB); // :vampire:
+	insert(0xD83E, 0xDDDB, 0x200D, 0x2640, 0xFE0F); // :woman_vampire:
+	insert(0xD83E, 0xDDDB, 0x200D, 0x2642, 0xFE0F); // :man_vampire:
+	insert(0xD83E, 0xDDDC); // :merperson:
+	insert(0xD83E, 0xDDDC, 0x200D, 0x2640, 0xFE0F); // :mermaid:
+	insert(0xD83E, 0xDDDC, 0x200D, 0x2642, 0xFE0F); // :merman:
+	insert(0xD83E, 0xDDDD); // :elf:
+	insert(0xD83E, 0xDDDD, 0x200D, 0x2640, 0xFE0F); // :woman_elf:
+	insert(0xD83E, 0xDDDD, 0x200D, 0x2642, 0xFE0F); // :man_elf:
+	insert(0xD83E, 0xDDDE); // :genie:
+	insert(0xD83E, 0xDDDE, 0x200D, 0x2640, 0xFE0F); // :woman_genie:
+	insert(0xD83E, 0xDDDE, 0x200D, 0x2642, 0xFE0F); // :man_genie:
+	insert(0xD83E, 0xDDDF); // :zombie:
+	insert(0xD83E, 0xDDDF, 0x200D, 0x2640, 0xFE0F); // :woman_zombie:
+	insert(0xD83E, 0xDDDF, 0x200D, 0x2642, 0xFE0F); // :man_zombie:
+	insert(0xD83E, 0xDDE0); // :brain:
+	insert(0xD83E, 0xDDE1); // :orange_heart:
+	insert(0xD83E, 0xDDE2); // :billed_cap:
+	insert(0xD83E, 0xDDE3); // :scarf:
+	insert(0xD83E, 0xDDE4); // :gloves:
+	insert(0xD83E, 0xDDE5); // :coat:
+	insert(0xD83E, 0xDDE6); // :socks:
+
+	insert(0x23CF, 0xFE0F); // :eject:
+
+	insert(0x2640, 0xFE0F); // :female_sign:
+	insert(0x2642, 0xFE0F); // :male_sign:
+	insert(0x2695, 0xFE0F); // :medical_symbol:
+
+	return result;
+})();
+
+const auto ConvertMap = ([] {
+	auto result = QMap<QString, QString>();
+	auto insert = [&result](const std::initializer_list<QChar> &from, const std::initializer_list<QChar> &to) {
+		result.insert(ComposeString(from), ComposeString(to));
+	};
+	auto insertWithAdd = [&result](const std::initializer_list<QChar> &from, const QString &added) {
+		auto code = ComposeString(from);
+		result.insert(code, code + added);
+	};
+	auto maleModifier = ComposeString({ 0x200D, 0x2642, 0xFE0F });
+	auto femaleModifier = ComposeString({ 0x200D, 0x2640, 0xFE0F });
+	insertWithAdd({ 0xD83E, 0xDD26 }, maleModifier);
+	insertWithAdd({ 0xD83E, 0xDD37 }, femaleModifier);
+	insertWithAdd({ 0xD83E, 0xDD38 }, maleModifier);
+	insertWithAdd({ 0xD83E, 0xDD39 }, maleModifier);
+	insertWithAdd({ 0xD83E, 0xDD3C }, maleModifier);
+	insertWithAdd({ 0xD83E, 0xDD3D }, maleModifier);
+	insertWithAdd({ 0xD83E, 0xDD3E }, femaleModifier);
+
+	// :kiss_woman_man:
+	insert({ 0xD83D, 0xDC69, 0x200D, 0x2764, 0xFE0F, 0x200D, 0xD83D, 0xDC8B, 0x200D, 0xD83D, 0xDC68 }, { 0xD83D, 0xDC8F });
+
+	// :family_man_woman_boy:
+	insert({ 0xD83D, 0xDC68, 0x200D, 0xD83D, 0xDC69, 0x200D, 0xD83D, 0xDC66 }, { 0xD83D, 0xDC6A });
+
+	// :couple_with_heart_woman_man:
+	insert({ 0xD83D, 0xDC69, 0x200D, 0x2764, 0xFE0F, 0x200D, 0xD83D, 0xDC68 }, { 0xD83D, 0xDC91 });
+
+	auto insertFlag = [insert](char ch1, char ch2, char ch3, char ch4) {
+		insert({ 0xD83C, 0xDDE6 + (ch1 - 'a'), 0xD83C, 0xDDe6 + (ch2 - 'a') }, { 0xD83C, 0xDDE6 + (ch3 - 'a'), 0xD83C, 0xDDe6 + (ch4 - 'a') });
+	};
+	insertFlag('a', 'c', 's', 'h');
+	insertFlag('b', 'v', 'n', 'o');
+	insertFlag('c', 'p', 'f', 'r');
+	insertFlag('d', 'g', 'i', 'o');
+	insertFlag('e', 'a', 'e', 's');
+	insertFlag('h', 'm', 'a', 'u');
+	insertFlag('m', 'f', 'f', 'r');
+	insertFlag('s', 'j', 'n', 'o');
+	insertFlag('t', 'a', 's', 'h');
+	insertFlag('u', 'm', 'u', 's');
+
+	return result;
+})();
+
+// Empty string result means we should skip this one.
+QString ConvertEmojiId(const Id &id, const QString &replacement) {
+	if (RegExpTone.match(replacement).hasMatch()) {
+		return QString();
+	}
+	if (NotSupported.contains(id)) {
+		return QString();
+	}
+	if (QRegularExpression("_tone").match(replacement).hasMatch()) {
+		int a = 0;
+	}
+	return ConvertMap.value(id, id);
+}
+
+} // namespace
+
+Replaces PrepareReplaces(const QString &filename) {
+	auto result = Replaces(filename);
+	auto content = ([filename] {
+		QFile f(filename);
+		return f.open(QIODevice::ReadOnly) ? f.readAll() : QByteArray();
+	})();
+	if (content.isEmpty()) {
+		logReplacesError(filename) << "Could not read data.";
+		return result;
+	}
+	auto error = QJsonParseError();
+	auto document = QJsonDocument::fromJson(content, &error);
+	if (error.error != QJsonParseError::NoError) {
+		logReplacesError(filename) << "Could not parse data (" << int(error.error) << "): " << error.errorString().toStdString();
+		return result;
+	}
+	if (!document.isObject()) {
+		logReplacesError(filename) << "Root object not found.";
+		return result;
+	}
+	auto list = document.object();
+	for (auto i = list.constBegin(), e = list.constEnd(); i != e; ++i) {
+		if (!i->isObject()) {
+			logReplacesError(filename) << "Child object not found.";
+			return Replaces(filename);
+		}
+		auto childKey = i.key();
+		auto child = i->toObject();
+		auto failed = false;
+		auto getString = [filename, childKey, &child, &failed](const QString &key) {
+			auto it = child.constFind(key);
+			if (it == child.constEnd() || !it->isString()) {
+				logReplacesError(filename) << "Child '" << childKey.toStdString() << "' field not found: " << key.toStdString();
+				failed = true;
+				return QString();
+			}
+			return it->toString();
+		};
+		auto idParts = getString("output").split('-');
+		auto name = getString("name");
+		auto replacement = getString("alpha_code");
+		auto aliases = getString("aliases").split('|');
+		if (aliases.size() == 1 && aliases[0].isEmpty()) {
+			aliases.clear();
+		}
+		if (failed) {
+			return Replaces(filename);
+		}
+		if (!RegExpCode.match(replacement).hasMatch()) {
+			logReplacesError(filename) << "Child '" << childKey.toStdString() << "' alpha_code invalid: " << replacement.toStdString();
+			return Replaces(filename);
+		}
+		for (auto &alias : aliases) {
+			if (!RegExpCode.match(alias).hasMatch()) {
+				logReplacesError(filename) << "Child '" << childKey.toStdString() << "' alias invalid: " << alias.toStdString();
+				return Replaces(filename);
+			}
+		}
+		auto id = Id();
+		for (auto &idPart : idParts) {
+			auto ok = true;
+			auto utf32 = idPart.toInt(&ok, 0x10);
+			if (!ok || !RegExpHex.match(idPart).hasMatch()) {
+				logReplacesError(filename) << "Child '" << childKey.toStdString() << "' output part invalid: " << idPart.toStdString();
+				return Replaces(filename);
+			}
+			if (utf32 >= 0 && utf32 < 0x10000) {
+				auto ch = QChar(ushort(utf32));
+				if (ch.isLowSurrogate() || ch.isHighSurrogate()) {
+					logReplacesError(filename) << "Child '" << childKey.toStdString() << "' output part invalid: " << idPart.toStdString();
+					return Replaces(filename);
+				}
+				id.append(ch);
+			} else if (utf32 >= 0x10000 && utf32 <= 0x10FFFF) {
+				auto hi = ((utf32 - 0x10000) / 0x400) + 0xD800;
+				auto lo = ((utf32 - 0x10000) % 0x400) + 0xDC00;
+				id.append(QChar(ushort(hi)));
+				id.append(QChar(ushort(lo)));
+			} else {
+				logReplacesError(filename) << "Child '" << childKey.toStdString() << "' output part invalid: " << idPart.toStdString();
+				return Replaces(filename);
+			}
+		}
+		id = ConvertEmojiId(id, replacement);
+		if (id.isEmpty()) {
+			continue;
+		}
+		if (!AddReplacement(result, id, replacement, name)) {
+			return Replaces(filename);
+		}
+		for (auto &alias : aliases) {
+			if (!AddReplacement(result, id, alias, name)) {
+				return Replaces(filename);
+			}
+		}
+	}
+	if (!AddReplacement(result, ComposeString({ 0xD83D, 0xDC4D }), ":like:", "thumbs up")) {
+		return Replaces(filename);
+	}
+	if (!AddReplacement(result, ComposeString({ 0xD83D, 0xDC4E }), ":dislike:", "thumbs down")) {
+		return Replaces(filename);
+	}
+	if (!AddReplacement(result, ComposeString({ 0xD83E, 0xDD14 }), ":hmm:", "thinking")) {
+		return Replaces(filename);
+	}
+	return result;
+}
+
+bool CheckAndConvertReplaces(Replaces &replaces, const Data &data) {
+	auto result = Replaces(replaces.filename);
+	auto sorted = QMap<Id, Replace>();
+	auto findId = [&data](const Id &id) {
+		return data.map.find(id) != data.map.cend();
+	};
+	auto findAndSort = [findId, &sorted](Id id, const Replace &replace) {
+		if (!findId(id)) {
+			id.replace(QChar(0xFE0F), QString());
+			if (!findId(id)) {
+				return false;
+			}
+		}
+		auto it = sorted.insertMulti(id, replace);
+		it.value().id = id;
+		return true;
+	};
+
+	// Find all replaces in data.map, adjust id if necessary.
+	// Store all replaces in sorted map to find them fast afterwards.
+	auto maleModifier = ComposeString({ 0x200D, 0x2642, 0xFE0F });
+	auto femaleModifier = ComposeString({ 0x200D, 0x2640, 0xFE0F });
+	for (auto &replace : replaces.list) {
+		if (findAndSort(replace.id, replace)) {
+			continue;
+		}
+		if (replace.id.endsWith(maleModifier)) {
+			auto defaultId = replace.id.mid(0, replace.id.size() - maleModifier.size());
+			if (findAndSort(defaultId, replace)) {
+				continue;
+			}
+		} else if (replace.id.endsWith(femaleModifier)) {
+			auto defaultId = replace.id.mid(0, replace.id.size() - femaleModifier.size());
+			if (findAndSort(defaultId, replace)) {
+				continue;
+			}
+		} else if (findId(replace.id + maleModifier)) {
+			if (findId(replace.id + femaleModifier)) {
+				logReplacesError(replaces.filename) << "Replace '" << replace.replacement.toStdString() << "' ambiguous.";
+				return false;
+			} else {
+				findAndSort(replace.id + maleModifier, replace);
+				continue;
+			}
+		} else if (findAndSort(replace.id + femaleModifier, replace)) {
+			continue;
+		}
+		logReplacesError(replaces.filename) << "Replace '" << replace.replacement.toStdString() << "' not found.";
+		return false;
+	}
+
+	// Go through all categories and put all replaces in order of emoji in categories.
+	result.list.reserve(replaces.list.size());
+	for (auto &category : data.categories) {
+		for (auto index : category) {
+			auto id = data.list[index].id;
+			auto found = false;
+			for (auto it = sorted.find(id); it != sorted.cend(); sorted.erase(it), it = sorted.find(id)) {
+				found = true;
+				result.list.push_back(it.value());
+			}
+			id.replace(QChar(0xFE0F), QString());
+			for (auto it = sorted.find(id); it != sorted.cend(); sorted.erase(it), it = sorted.find(id)) {
+				if (found) {
+					logReplacesError(replaces.filename) << "Strange emoji, found in both ways: " << it->replacement.toStdString();
+					return false;
+				}
+				result.list.push_back(it.value());
+			}
+		}
+	}
+	if (result.list.size() != replaces.list.size()) {
+		logReplacesError(replaces.filename) << "Some were not found.";
+		return false;
+	}
+	if (!sorted.isEmpty()) {
+		logReplacesError(replaces.filename) << "Weird.";
+		return false;
+	}
+	replaces = std::move(result);
+	return true;
+}
+
+} // namespace emoji
+} // namespace codegen

Telegram/SourceFiles/codegen/emoji/replaces.h

@@ -0,0 +1,47 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop version of Telegram messaging app, see https://telegram.org
+
+Telegram Desktop is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+It is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+In addition, as a special exception, the copyright holders give permission
+to link the code of portions of this program with the OpenSSL library.
+
+Full license: https://github.com/telegramdesktop/tdesktop/blob/master/LICENSE
+Copyright (c) 2014-2017 John Preston, https://desktop.telegram.org
+*/
+#pragma once
+
+#include "codegen/common/logging.h"
+#include "codegen/emoji/data.h"
+#include <QtCore/QVector>
+
+namespace codegen {
+namespace emoji {
+
+struct Replace {
+	Id id;
+	QString replacement;
+	QVector<QString> words;
+};
+
+struct Replaces {
+	Replaces(const QString &filename) : filename(filename) {
+	}
+	QString filename;
+	QVector<Replace> list;
+};
+
+Replaces PrepareReplaces(const QString &filename);
+bool CheckAndConvertReplaces(Replaces &replaces, const Data &data);
+
+} // namespace emoji
+} // namespace codegen

Telegram/gyp/codegen.gyp

@@ -162,6 +162,8 @@
       '<(src_loc)/codegen/emoji/main.cpp',
       '<(src_loc)/codegen/emoji/options.cpp',
       '<(src_loc)/codegen/emoji/options.h',
+      '<(src_loc)/codegen/emoji/replaces.cpp',
+      '<(src_loc)/codegen/emoji/replaces.h',
     ],
   }],
 }

Telegram/gyp/codegen_rules.gypi

@@ -131,6 +131,7 @@
     'action_name': 'codegen_emoji',
     'inputs': [
       '<(PRODUCT_DIR)/codegen_emoji<(exe_ext)',
+      '<(res_loc)/emoji_autocomplete.json',
     ],
     'outputs': [
       '<(SHARED_INTERMEDIATE_DIR)/emoji.cpp',
@@ -138,6 +139,7 @@
     ],
     'action': [
       '<(PRODUCT_DIR)/codegen_emoji<(exe_ext)',
+      '<(res_loc)/emoji_autocomplete.json',
       '-o', '<(SHARED_INTERMEDIATE_DIR)',
     ],
     'message': 'codegen_emoji-ing..',

Telegram/gyp/telegram_sources.txt

@@ -98,6 +98,10 @@
 <(src_loc)/chat_helpers/bot_keyboard.h
 <(src_loc)/chat_helpers/emoji_list_widget.cpp
 <(src_loc)/chat_helpers/emoji_list_widget.h
+<(src_loc)/chat_helpers/emoji_suggestions.cpp
+<(src_loc)/chat_helpers/emoji_suggestions.h
+<(src_loc)/chat_helpers/emoji_suggestions_widget.cpp
+<(src_loc)/chat_helpers/emoji_suggestions_widget.h
 <(src_loc)/chat_helpers/field_autocomplete.cpp
 <(src_loc)/chat_helpers/field_autocomplete.h
 <(src_loc)/chat_helpers/gifs_list_widget.cpp