Add basic implementation of Storage::Cache DB.

Telegram/SourceFiles/base/build_config.h

@@ -56,11 +56,11 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 #endif
 
 #if defined(__GNUC__)
-#define FORCE_INLINE inline __attribute__((always_inline))
+#define TG_FORCE_INLINE inline __attribute__((always_inline))
 #elif defined(_MSC_VER)
-#define FORCE_INLINE __forceinline
+#define TG_FORCE_INLINE __forceinline
 #else
-#define FORCE_INLINE inline
+#define TG_FORCE_INLINE inline
 #endif
 
 #include <climits>

Telegram/SourceFiles/base/unique_function.h

@@ -124,12 +124,8 @@ public:
 		_impl.swap(other._impl);
 	}
 
-	template <
+	Return operator()(Args ...args) {
-		typename ...OtherArgs,
+		return _impl(std::forward<Args>(args)...);
-		typename = decltype(std::declval<std::function<Return(Args...)>>()(
-			std::declval<OtherArgs>()...))>
-	Return operator()(OtherArgs &&...args) {
-		return _impl(std::forward<OtherArgs>(args)...);
 	}
 
 	explicit operator bool() const {

Telegram/SourceFiles/core/basic_types.h

@@ -32,6 +32,8 @@ template <typename Signature>
 using FnMut = base::unique_function<Signature>;
 
 //using uchar = unsigned char; // Qt has uchar
+using int8 = qint8;
+using uint8 = quint8;
 using int16 = qint16;
 using uint16 = quint16;
 using int32 = qint32;

Telegram/SourceFiles/media/media_audio.h

@@ -324,11 +324,11 @@ VoiceWaveform audioCountWaveform(const FileLocation &file, const QByteArray &dat
 namespace Media {
 namespace Audio {
 
-FORCE_INLINE uint16 ReadOneSample(uchar data) {
+TG_FORCE_INLINE uint16 ReadOneSample(uchar data) {
 	return qAbs((static_cast<int16>(data) - 0x80) * 0x100);
 }
 
-FORCE_INLINE uint16 ReadOneSample(int16 data) {
+TG_FORCE_INLINE uint16 ReadOneSample(int16 data) {
 	return qAbs(data);
 }
 

Telegram/SourceFiles/stdafx.h

@@ -6,7 +6,6 @@ For license and copyright information please follow this link:
 https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 */
 
-#define NOMINMAX // no min() and max() macro declarations
 #define __HUGE
 
 // Fix Google Breakpad build for Mac App Store version

Telegram/SourceFiles/storage/cache/storage_cache_database.cpp

@@ -7,3 +7,733 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 */
 #include "storage/cache/storage_cache_database.h"
 
+#include "storage/storage_encryption.h"
+#include "storage/storage_encrypted_file.h"
+#include "base/flat_set.h"
+#include "base/algorithm.h"
+#include <crl/crl.h>
+#include <xxhash.h>
+#include <QtCore/QDir>
+#include <unordered_map>
+#include <set>
+
+namespace std {
+
+template <>
+struct hash<Storage::Cache::Key> {
+	size_t operator()(const Storage::Cache::Key &key) const {
+		return (hash<uint64>()(key.high) ^ hash<uint64>()(key.low));
+	}
+};
+
+} // namespace std
+
+namespace Storage {
+namespace Cache {
+namespace details {
+namespace {
+
+constexpr auto kMaxBundledRecords = 256 * 1024;
+constexpr auto kReadBlockSize = 8 * 1024 * 1024;
+constexpr auto kRecordSizeUnknown = size_type(-1);
+constexpr auto kRecordSizeInvalid = size_type(-2);
+constexpr auto kMaxDataSize = 10 * 1024 * 1024;
+
+using RecordType = uint8;
+using PlaceId = std::array<uint8, 7>;
+using EntrySize = std::array<uint8, 3>;
+using RecordsCount = std::array<uint8, 3>;
+
+static_assert(kMaxBundledRecords < (1 << (RecordsCount().size() * 8)));
+static_assert(kMaxDataSize < (1 << (EntrySize().size() * 8)));
+
+template <typename Packed>
+Packed ReadTo(size_type count) {
+	Expects(count >= 0 && count < (1 << (Packed().size() * 8)));
+
+	auto result = Packed();
+	for (auto &element : result) {
+		element = uint8(count & 0xFF);
+		count >>= 8;
+	}
+	return result;
+}
+
+template <typename Packed>
+size_type ReadFrom(Packed count) {
+	auto result = size_type();
+	RecordsCount a;
+	for (auto &element : (count | ranges::view::reverse)) {
+		result <<= 8;
+		result |= size_type(element);
+	}
+	return result;
+}
+
+uint32 CountChecksum(bytes::const_span data) {
+	const auto seed = uint32(0);
+	return XXH32(data.data(), data.size(), seed);
+}
+
+QString PlaceFromId(PlaceId place) {
+	auto result = QString();
+	result.reserve(15);
+	const auto pushDigit = [&](uint8 digit) {
+		const auto hex = (digit < 0x0A)
+			? char('0' + digit)
+			: char('A' + (digit - 0x0A));
+		result.push_back(hex);
+	};
+	const auto push = [&](uint8 value) {
+		pushDigit(value & 0x0F);
+		pushDigit(value >> 4);
+	};
+	for (auto i = 0; i != place.size(); ++i) {
+		push(place[i]);
+		if (i == 1) {
+			result.push_back('/');
+		}
+	}
+	return result;
+}
+
+struct Store {
+	static constexpr auto kType = RecordType(0x01);
+
+	RecordType type = kType;
+	uint8 tag = 0;
+	PlaceId place = { { 0 } };
+	EntrySize size = { { 0 } };
+	uint32 checksum = 0;
+	Key key;
+};
+static_assert(sizeof(Store) == 1 + 7 + 1 + 3 + 4 + 16);
+
+struct MultiStoreHeader {
+	static constexpr auto kType = RecordType(0x02);
+
+	explicit MultiStoreHeader(size_type count = 0);
+
+	RecordType type = kType;
+	RecordsCount count = { { 0 } };
+};
+struct MultiStorePart {
+	uint8 reserved = 0;
+	PlaceId place = { { 0 } };
+	uint8 tag = 0;
+	EntrySize size = { { 0 } };
+	uint32 checksum = 0;
+	Key key;
+};
+static_assert(sizeof(MultiStoreHeader) == 4);
+static_assert(sizeof(MultiStorePart) == sizeof(Store));
+
+MultiStoreHeader::MultiStoreHeader(size_type count)
+: type(kType)
+, count(ReadTo<RecordsCount>(count)) {
+	Expects(count >= 0 && count < kMaxBundledRecords);
+}
+
+struct MultiRemoveHeader {
+	static constexpr auto kType = RecordType(0x03);
+
+	explicit MultiRemoveHeader(size_type count = 0);
+
+	RecordType type = kType;
+	RecordsCount count = { { 0 } };
+};
+struct MultiRemovePart {
+	Key key;
+};
+static_assert(sizeof(MultiRemoveHeader) == 4);
+static_assert(sizeof(MultiRemovePart) == 16);
+
+MultiRemoveHeader::MultiRemoveHeader(size_type count)
+: type(kType)
+, count(ReadTo<RecordsCount>(count)) {
+	Expects(count >= 0 && count < kMaxBundledRecords);
+}
+
+} // namespace
+
+class Database {
+public:
+	using Wrapper = Cache::Database;
+	using Settings = Wrapper::Settings;
+	Database(const QString &path, const Settings &settings);
+
+	using Error = Wrapper::Error;
+	void open(EncryptionKey key, FnMut<void(Error)> done);
+	void close(FnMut<void()> done);
+
+	void put(const Key &key, QByteArray value, FnMut<void(Error)> done);
+	void get(const Key &key, FnMut<void(QByteArray)> done);
+	void remove(const Key &key, FnMut<void()> done);
+
+	void clear(FnMut<void(Error)> done);
+
+private:
+	using Version = int32;
+	struct Entry {
+		Entry() = default;
+		Entry(PlaceId place, uint8 tag, uint32 checksum, size_type size);
+
+		uint64 tag = 0;
+		uint32 checksum = 0;
+		size_type size = 0;
+		PlaceId place = { { 0 } };
+	};
+
+	template <typename Callback, typename ...Args>
+	void invokeCallback(Callback &&callback, Args &&...args);
+
+	Error ioError(const QString &path) const;
+
+	QString computePath(Version version) const;
+	QString binlogPath(Version version) const;
+	QString binlogPath() const;
+	QString binlogFilename() const;
+	File::Result openBinlog(
+		Version version,
+		File::Mode mode,
+		EncryptionKey &key);
+	void readBinlog();
+	size_type readBinlogRecords(bytes::const_span data);
+	size_type readBinlogRecordSize(bytes::const_span data) const;
+	bool readBinlogRecord(bytes::const_span data);
+	bool readRecordStore(bytes::const_span data);
+	bool readRecordMultiStore(bytes::const_span data);
+	bool readRecordMultiRemove(bytes::const_span data);
+
+	Version findAvailableVersion() const;
+	QString versionPath() const;
+	bool writeVersion(Version version);
+	Version readVersion() const;
+
+	QString placePath(PlaceId place) const;
+	bool isFreePlace(PlaceId place) const;
+	QString writeKeyPlace(const Key &key, size_type size, uint32 checksum);
+	void writeMultiRemove();
+
+	QString _base, _path;
+	Settings _settings;
+	EncryptionKey _key;
+	File _binlog;
+	std::unordered_map<Key, Entry> _map;
+	std::set<Key> _removing;
+
+};
+
+Database::Entry::Entry(
+	PlaceId place,
+	uint8 tag,
+	uint32 checksum,
+	size_type size)
+: tag(tag)
+, checksum(checksum)
+, place(place)
+, size(size) {
+}
+
+Database::Database(const QString &path, const Settings &settings)
+: _base(QDir(path).absolutePath() + '/')
+, _settings(settings) {
+}
+
+template <typename Callback, typename ...Args>
+void Database::invokeCallback(Callback &&callback, Args &&...args) {
+	if (callback) {
+		callback(std::move(args)...);
+		//crl::on_main([
+		//	callback = std::move(callback),
+		//	args = std::forward<Args>(args)...
+		//]() mutable {
+		//	callback(std::move(args)...);
+		//});
+	}
+}
+
+auto Database::ioError(const QString &path) const -> Error {
+	return { Error::Type::IO, path };
+}
+
+void Database::open(EncryptionKey key, FnMut<void(Error)> done) {
+	const auto version = readVersion();
+	const auto result = openBinlog(version, File::Mode::ReadAppend, key);
+	switch (result) {
+	case File::Result::Success:
+		invokeCallback(done, Error::NoError());
+		break;
+	case File::Result::LockFailed:
+		invokeCallback(
+			done,
+			Error{ Error::Type::LockFailed, binlogPath(version) });
+		break;
+	case File::Result::WrongKey:
+		invokeCallback(
+			done,
+			Error{ Error::Type::WrongKey, binlogPath(version) });
+		break;
+	case File::Result::Failed: {
+		const auto available = findAvailableVersion();
+		const auto retry = openBinlog(available, File::Mode::Write, key);
+		if (retry == File::Result::Success) {
+			if (writeVersion(available)) {
+				invokeCallback(done, Error::NoError());
+			} else {
+				invokeCallback(done, ioError(versionPath()));
+			}
+		} else {
+			invokeCallback(done, ioError(binlogPath(available)));
+		}
+	} break;
+	default: Unexpected("Result from Database::openBinlog.");
+	}
+}
+
+QString Database::computePath(Version version) const {
+	return _base + QString::number(version) + '/';
+}
+
+QString Database::binlogFilename() const {
+	return qsl("binlog");
+}
+
+QString Database::binlogPath(Version version) const {
+	return computePath(version) + binlogFilename();
+}
+
+QString Database::binlogPath() const {
+	return _path + binlogFilename();
+}
+
+File::Result Database::openBinlog(
+		Version version,
+		File::Mode mode,
+		EncryptionKey &key) {
+	const auto path = binlogPath(version);
+	const auto result = _binlog.open(path, mode, key);
+	if (result == File::Result::Success) {
+		_path = computePath(version);
+		_key = std::move(key);
+		readBinlog();
+	}
+	return result;
+}
+
+void Database::readBinlog() {
+	auto data = bytes::vector(kReadBlockSize);
+	const auto full = bytes::make_span(data);
+	auto notParsedBytes = index_type(0);
+	while (true) {
+		Assert(notParsedBytes < full.size());
+		const auto readBytes = _binlog.read(full.subspan(notParsedBytes));
+		if (!readBytes) {
+			break;
+		}
+		notParsedBytes += readBytes;
+		const auto bytes = full.subspan(0, notParsedBytes);
+		const auto parsedTill = readBinlogRecords(bytes);
+		if (parsedTill == kRecordSizeInvalid) {
+			break;
+		}
+		Assert(parsedTill >= 0 && parsedTill <= notParsedBytes);
+		notParsedBytes -= parsedTill;
+		if (parsedTill > 0 && parsedTill < bytes.size()) {
+			bytes::move(full, bytes.subspan(parsedTill));
+		}
+	}
+	_binlog.seek(_binlog.offset() - notParsedBytes);
+}
+
+size_type Database::readBinlogRecords(bytes::const_span data) {
+	auto result = 0;
+	while (true) {
+		const auto size = readBinlogRecordSize(data);
+		if (size == kRecordSizeUnknown || size > data.size()) {
+			return result;
+		} else if (size == kRecordSizeInvalid || !readBinlogRecord(data)) {
+			return (result > 0) ? result : kRecordSizeInvalid;
+		} else {
+			result += size;
+			data = data.subspan(size);
+		}
+	}
+}
+
+size_type Database::readBinlogRecordSize(bytes::const_span data) const {
+	if (data.empty()) {
+		return kRecordSizeUnknown;
+	}
+
+	switch (static_cast<RecordType>(data[0])) {
+	case Store::kType:
+		return sizeof(Store);
+
+	case MultiStoreHeader::kType:
+		if (data.size() >= sizeof(MultiStoreHeader)) {
+			const auto header = reinterpret_cast<const MultiStoreHeader*>(
+				data.data());
+			const auto count = ReadFrom(header->count);
+			return (count > 0 && count < kMaxBundledRecords)
+				? (sizeof(MultiStoreHeader)
+					+ count * sizeof(MultiStorePart))
+				: kRecordSizeInvalid;
+		}
+		return kRecordSizeUnknown;
+
+	case MultiRemoveHeader::kType:
+		if (data.size() >= sizeof(MultiRemoveHeader)) {
+			const auto header = reinterpret_cast<const MultiRemoveHeader*>(
+				data.data());
+			const auto count = ReadFrom(header->count);
+			return (count > 0 && count < kMaxBundledRecords)
+				? (sizeof(MultiRemoveHeader)
+					+ count * sizeof(MultiRemovePart))
+				: kRecordSizeInvalid;
+		}
+		return kRecordSizeUnknown;
+
+	}
+	return kRecordSizeInvalid;
+}
+
+bool Database::readBinlogRecord(bytes::const_span data) {
+	Expects(!data.empty());
+
+	switch (static_cast<RecordType>(data[0])) {
+	case Store::kType:
+		return readRecordStore(data);
+
+	case MultiStoreHeader::kType:
+		return readRecordMultiStore(data);
+
+	case MultiRemoveHeader::kType:
+		return readRecordMultiRemove(data);
+
+	}
+	Unexpected("Bad type in Database::readBinlogRecord.");
+}
+
+bool Database::readRecordStore(bytes::const_span data) {
+	const auto record = reinterpret_cast<const Store*>(data.data());
+	const auto size = ReadFrom(record->size);
+	if (size > kMaxDataSize) {
+		return false;
+	}
+	_map[record->key] = Entry(
+		record->place,
+		record->tag,
+		record->checksum,
+		size);
+	return true;
+}
+
+bool Database::readRecordMultiStore(bytes::const_span data) {
+	const auto bytes = data.data();
+	const auto record = reinterpret_cast<const MultiStoreHeader*>(bytes);
+	const auto count = ReadFrom(record->count);
+	Assert(data.size() >= sizeof(MultiStoreHeader)
+		+ count * sizeof(MultiStorePart));
+	const auto parts = gsl::make_span(
+		reinterpret_cast<const MultiStorePart*>(
+			bytes + sizeof(MultiStoreHeader)),
+		count);
+	for (const auto &part : parts) {
+		const auto size = ReadFrom(part.size);
+		if (part.reserved != 0 || size > kMaxDataSize) {
+			return false;
+		}
+		_map[part.key] = Entry(part.place, part.tag, part.checksum, size);
+	}
+	return true;
+}
+
+bool Database::readRecordMultiRemove(bytes::const_span data) {
+	const auto bytes = data.data();
+	const auto record = reinterpret_cast<const MultiRemoveHeader*>(bytes);
+	const auto count = ReadFrom(record->count);
+	Assert(data.size() >= sizeof(MultiRemoveHeader)
+		+ count * sizeof(MultiRemovePart));
+	const auto parts = gsl::make_span(
+		reinterpret_cast<const MultiRemovePart*>(
+			bytes + sizeof(MultiRemoveHeader)),
+		count);
+	for (const auto &part : parts) {
+		_map.erase(part.key);
+	}
+	return true;
+}
+
+void Database::close(FnMut<void()> done) {
+	_binlog.close();
+	invokeCallback(done);
+}
+
+void Database::put(
+		const Key &key,
+		QByteArray value,
+		FnMut<void(Error)> done) {
+	const auto checksum = CountChecksum(bytes::make_span(value));
+	const auto path = writeKeyPlace(key, value.size(), checksum);
+	if (path.isEmpty()) {
+		invokeCallback(done, ioError(binlogPath()));
+		return;
+	}
+	File data;
+	const auto result = data.open(path, File::Mode::Write, _key);
+	switch (result) {
+	case File::Result::Failed:
+		invokeCallback(done, ioError(path));
+		break;
+
+	case File::Result::LockFailed:
+		invokeCallback(done, Error{ Error::Type::LockFailed, path });
+		break;
+
+	case File::Result::Success: {
+		const auto success = data.writeWithPadding(bytes::make_span(value));
+		if (!success) {
+			data.close();
+			remove(key, nullptr);
+			invokeCallback(done, ioError(path));
+		} else {
+			data.flush();
+			invokeCallback(done, Error::NoError());
+		}
+	} break;
+
+	default: Unexpected("Result in Database::put.");
+	}
+}
+
+QString Database::writeKeyPlace(
+		const Key &key,
+		size_type size,
+		uint32 checksum) {
+	Expects(size <= kMaxDataSize);
+
+	auto record = Store();
+	record.key = key;
+	record.size = ReadTo<EntrySize>(size);
+	record.checksum = checksum;
+	do {
+		bytes::set_random(bytes::object_as_span(&record.place));
+	} while (!isFreePlace(record.place));
+
+	const auto result = placePath(record.place);
+	auto writeable = record;
+	const auto success = _binlog.write(bytes::object_as_span(&writeable));
+	if (!success) {
+		return QString();
+	}
+	_binlog.flush();
+	readRecordStore(bytes::object_as_span(&record));
+	return result;
+}
+
+void Database::get(const Key &key, FnMut<void(QByteArray)> done) {
+	const auto i = _map.find(key);
+	if (i == _map.end()) {
+		invokeCallback(done, QByteArray());
+		return;
+	}
+	const auto &entry = i->second;
+	const auto path = placePath(entry.place);
+	File data;
+	const auto result = data.open(path, File::Mode::Read, _key);
+	switch (result) {
+	case File::Result::Failed:
+		invokeCallback(done, QByteArray());
+		break;
+
+	case File::Result::WrongKey:
+		invokeCallback(done, QByteArray());
+		break;
+
+	case File::Result::Success: {
+		auto result = QByteArray(entry.size, Qt::Uninitialized);
+		const auto bytes = bytes::make_span(result);
+		const auto read = data.readWithPadding(bytes);
+		if (read != entry.size || CountChecksum(bytes) != entry.checksum) {
+			invokeCallback(done, QByteArray());
+		} else {
+			invokeCallback(done, std::move(result));
+		}
+	} break;
+
+	default: Unexpected("Result in Database::get.");
+	}
+}
+
+void Database::remove(const Key &key, FnMut<void()> done) {
+	const auto i = _map.find(key);
+	if (i != _map.end()) {
+		_removing.emplace(key);
+		if (true || _removing.size() == kMaxBundledRecords) {
+			writeMultiRemove();
+			// cancel timeout?..
+		} else {
+			// timeout?..
+		}
+
+		const auto &entry = i->second;
+		const auto path = placePath(entry.place);
+		_map.erase(i);
+		QFile(path).remove();
+	}
+	invokeCallback(done);
+}
+
+void Database::writeMultiRemove() {
+	Expects(_removing.size() <= kMaxBundledRecords);
+
+	if (_removing.empty()) {
+		return;
+	}
+	const auto size = _removing.size();
+	auto header = MultiRemoveHeader(size);
+	auto list = std::vector<MultiRemovePart>();
+	list.reserve(size);
+	for (const auto &key : base::take(_removing)) {
+		list.push_back({ key });
+	}
+	if (_binlog.write(bytes::object_as_span(&header))) {
+		_binlog.write(bytes::make_span(list));
+		_binlog.flush();
+	}
+}
+
+void Database::clear(FnMut<void(Error)> done) {
+	Expects(_key.empty());
+
+	const auto version = findAvailableVersion();
+	invokeCallback(
+		done,
+		writeVersion(version) ? Error::NoError() : ioError(versionPath()));
+}
+
+auto Database::findAvailableVersion() const -> Version {
+	const auto entries = QDir(_base).entryList(
+		QDir::Dirs | QDir::NoDotAndDotDot);
+	auto versions = base::flat_set<Version>();
+	for (const auto entry : entries) {
+		versions.insert(entry.toInt());
+	}
+	auto result = Version();
+	for (const auto version : versions) {
+		if (result != version) {
+			break;
+		}
+		++result;
+	}
+	return result;
+}
+
+QString Database::versionPath() const {
+	return _base + "version";
+}
+
+bool Database::writeVersion(Version version) {
+	const auto bytes = QByteArray::fromRawData(
+		reinterpret_cast<const char*>(&version),
+		sizeof(version));
+
+	if (!QDir().mkpath(_base)) {
+		return false;
+	}
+	QFile file(versionPath());
+	if (!file.open(QIODevice::WriteOnly)) {
+		return false;
+	} else if (file.write(bytes) != bytes.size()) {
+		return false;
+	}
+	return file.flush();
+}
+
+auto Database::readVersion() const -> Version {
+	QFile file(versionPath());
+	if (!file.open(QIODevice::ReadOnly)) {
+		return Version();
+	}
+	const auto bytes = file.read(sizeof(Version));
+	if (bytes.size() != sizeof(Version)) {
+		return Version();
+	}
+	return *reinterpret_cast<const Version*>(bytes.data());
+}
+
+QString Database::placePath(PlaceId place) const {
+	return _path + PlaceFromId(place);
+}
+
+bool Database::isFreePlace(PlaceId place) const {
+	return !QFile(placePath(place)).exists();
+}
+
+} // namespace details
+
+Database::Database(const QString &path, const Settings &settings)
+: _wrapped(path, settings) {
+}
+
+void Database::open(EncryptionKey key, FnMut<void(Error)> done) {
+	_wrapped.with([
+		key,
+		done = std::move(done)
+	](Implementation &unwrapped) mutable {
+		unwrapped.open(key, std::move(done));
+	});
+}
+
+void Database::close(FnMut<void()> done) {
+	_wrapped.with([
+		done = std::move(done)
+	](Implementation &unwrapped) mutable {
+		unwrapped.close(std::move(done));
+	});
+}
+
+void Database::put(
+		const Key &key,
+		QByteArray value,
+		FnMut<void(Error)> done) {
+	_wrapped.with([
+		key,
+		value = std::move(value),
+		done = std::move(done)
+	](Implementation &unwrapped) mutable {
+		unwrapped.put(key, std::move(value), std::move(done));
+	});
+}
+
+void Database::get(const Key &key, FnMut<void(QByteArray)> done) {
+	_wrapped.with([
+		key,
+		done = std::move(done)
+	](Implementation &unwrapped) mutable {
+		unwrapped.get(key, std::move(done));
+	});
+}
+
+void Database::remove(const Key &key, FnMut<void()> done) {
+	_wrapped.with([
+		key,
+		done = std::move(done)
+	](Implementation &unwrapped) mutable {
+		unwrapped.remove(key, std::move(done));
+	});
+}
+
+void Database::clear(FnMut<void(Error)> done) {
+	_wrapped.with([
+		done = std::move(done)
+	](Implementation &unwrapped) mutable {
+		unwrapped.clear(std::move(done));
+	});
+}
+
+Database::~Database() = default;
+
+} // namespace Cache
+} // namespace Storage

Telegram/SourceFiles/storage/cache/storage_cache_database.h

@@ -8,9 +8,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 #pragma once
 
 #include "core/basic_types.h"
-
 #include <crl/crl_object_on_queue.h>
-
 #include <QtCore/QString>
 
 namespace Storage {
@@ -20,9 +18,54 @@ namespace details {
 class Database;
 } // namespace details
 
+struct Key {
+	uint64 high = 0;
+	uint64 low = 0;
+};
+
+inline bool operator==(const Key &a, const Key &b) {
+	return (a.high == b.high) && (a.low == b.low);
+}
+
+inline bool operator!=(const Key &a, const Key &b) {
+	return !(a == b);
+}
+
+inline bool operator<(const Key &a, const Key &b) {
+	return std::tie(a.high, a.low) < std::tie(b.high, b.low);
+}
+
 class Database {
 public:
-	Database(const QString &path, EncryptionKey key);
+	struct Settings {
+		size_type sizeLimit = 0;
+	};
+	Database(const QString &path, const Settings &settings);
+
+	struct Error {
+		enum class Type {
+			None,
+			IO,
+			WrongKey,
+			LockFailed,
+		};
+		Type type = Type::None;
+		QString path;
+
+		static Error NoError() {
+			return Error();
+		}
+	};
+	void open(EncryptionKey key, FnMut<void(Error)> done);
+	void close(FnMut<void()> done);
+
+	void put(const Key &key, QByteArray value, FnMut<void(Error)> done);
+	void get(const Key &key, FnMut<void(QByteArray)> done);
+	void remove(const Key &key, FnMut<void()> done);
+
+	void clear(FnMut<void(Error)> done);
+
+	~Database();
 
 private:
 	using Implementation = details::Database;

Telegram/SourceFiles/storage/cache/storage_cache_database_tests.cpp

@@ -0,0 +1,106 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "catch.hpp"
+
+#include "storage/cache/storage_cache_database.h"
+#include "storage/storage_encryption.h"
+
+#include <crl/crl.h>
+
+using namespace Storage::Cache;
+
+const auto key = Storage::EncryptionKey(bytes::make_vector(
+	bytes::make_span("\
+abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\
+abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\
+abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\
+abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\
+").subspan(0, Storage::EncryptionKey::kSize)));
+
+const auto name = QString("test.db");
+
+const auto TestValue1 = QByteArray("testbytetestbyt");
+const auto TestValue2 = QByteArray("bytetestbytetestb");
+
+crl::semaphore Semaphore;
+
+auto Result = Database::Error();
+const auto GetResult = [](Database::Error error) {
+	Result = error;
+	Semaphore.release();
+};
+
+auto Value = QByteArray();
+const auto GetValue = [](QByteArray value) {
+	Value = value;
+	Semaphore.release();
+};
+
+const auto Settings = Database::Settings{ 1024 };
+
+TEST_CASE("encrypted cache db", "[storage_cache_database]") {
+	SECTION("writing db") {
+		Database db(name, Settings);
+
+		db.clear(GetResult);
+		Semaphore.acquire();
+		REQUIRE(Result.type == Database::Error::Type::None);
+
+		db.open(key, GetResult);
+		Semaphore.acquire();
+		REQUIRE(Result.type == Database::Error::Type::None);
+
+		db.put(Key{ 0, 1 }, TestValue1, GetResult);
+		Semaphore.acquire();
+		REQUIRE(Result.type == Database::Error::Type::None);
+
+		db.close([&] { Semaphore.release(); });
+		Semaphore.acquire();
+	}
+	SECTION("reading and writing db") {
+		Database db(name, Settings);
+
+		db.open(key, GetResult);
+		Semaphore.acquire();
+		REQUIRE(Result.type == Database::Error::Type::None);
+
+		db.get(Key{ 0, 1 }, GetValue);
+		Semaphore.acquire();
+		REQUIRE((Value == TestValue1));
+
+		db.put(Key{ 1, 0 }, TestValue2, GetResult);
+		Semaphore.acquire();
+		REQUIRE(Result.type == Database::Error::Type::None);
+
+		db.get(Key{ 1, 0 }, GetValue);
+		Semaphore.acquire();
+		REQUIRE((Value == TestValue2));
+
+		db.get(Key{ 1, 1 }, GetValue);
+		Semaphore.acquire();
+		REQUIRE(Value.isEmpty());
+
+		db.close([&] { Semaphore.release(); });
+		Semaphore.acquire();
+	}
+	SECTION("reading db") {
+		Database db(name, Settings);
+
+		db.open(key, GetResult);
+		Semaphore.acquire();
+		REQUIRE(Result.type == Database::Error::Type::None);
+
+		db.get(Key{ 0, 1 }, GetValue);
+		Semaphore.acquire();
+		REQUIRE((Value == TestValue1));
+
+		db.get(Key{ 1, 0 }, GetValue);
+		Semaphore.acquire();
+		REQUIRE((Value == TestValue2));
+	}
+}

Telegram/SourceFiles/storage/storage_encrypted_file.cpp

@@ -30,7 +30,15 @@ File::Result File::open(
 		const EncryptionKey &key) {
 	close();
 
-	_data.setFileName(QFileInfo(path).absoluteFilePath());
+	const auto info = QFileInfo(path);
+	const auto dir = info.absoluteDir();
+	if (mode != Mode::Read && !dir.exists()) {
+		if (!QDir().mkpath(dir.absolutePath())) {
+			return Result::Failed;
+		}
+	}
+
+	_data.setFileName(info.absoluteFilePath());
 	const auto result = attemptOpen(mode, key);
 	if (result != Result::Success) {
 		close();
@@ -105,6 +113,7 @@ bool File::writeHeader(const EncryptionKey &key) {
 	} else if (!write(headerBytes.subspan(header.salt.size()))) {
 		return false;
 	}
+	_dataSize = 0;
 	return true;
 }
 
@@ -132,6 +141,13 @@ File::Result File::readHeader(const EncryptionKey &key) {
 	} else if (header.format != Format::Format_0) {
 		return Result::Failed;
 	}
+	_dataSize = _data.size()
+		- int64(sizeof(BasicHeader))
+		- FileLock::kSkipBytes;
+	Assert(_dataSize >= 0);
+	if (const auto bad = (_dataSize % kBlockSize)) {
+		_dataSize -= bad;
+	}
 	return Result::Success;
 }
 
@@ -148,15 +164,15 @@ size_type File::writePlain(bytes::const_span bytes) {
 void File::decrypt(bytes::span bytes) {
 	Expects(_state.has_value());
 
-	_state->decrypt(bytes, _offset);
+	_state->decrypt(bytes, _encryptionOffset);
-	_offset += bytes.size();
+	_encryptionOffset += bytes.size();
 }
 
 void File::encrypt(bytes::span bytes) {
 	Expects(_state.has_value());
 
-	_state->encrypt(bytes, _offset);
+	_state->encrypt(bytes, _encryptionOffset);
-	_offset += bytes.size();
+	_encryptionOffset += bytes.size();
 }
 
 size_type File::read(bytes::span bytes) {
@@ -175,31 +191,108 @@ size_type File::read(bytes::span bytes) {
 	return count;
 }
 
-size_type File::write(bytes::span bytes) {
+bool File::write(bytes::span bytes) {
 	Expects(bytes.size() % kBlockSize == 0);
 
 	encrypt(bytes);
-	auto count = writePlain(bytes);
+	const auto count = writePlain(bytes);
-	if (const auto back = (count % kBlockSize)) {
+	if (count != bytes.size()) {
-		if (!_data.seek(_data.pos() - back)) {
+		decryptBack(bytes);
-			return 0;
+		if (count > 0) {
+			_data.seek(_data.pos() - count);
 		}
-		count -= back;
+		return false;
 	}
-	if (const auto back = (bytes.size() - count)) {
+	return true;
-		_offset -= back;
+}
-		decrypt(bytes.subspan(count));
+
+void File::decryptBack(bytes::span bytes) {
+	Expects(_encryptionOffset >= bytes.size());
+
+	_encryptionOffset -= bytes.size();
+	decrypt(bytes);
+	_encryptionOffset -= bytes.size();
+}
+
+size_type File::readWithPadding(bytes::span bytes) {
+	const auto size = bytes.size();
+	const auto part = size % kBlockSize;
+	const auto good = size - part;
+	if (good) {
+		const auto succeed = read(bytes.subspan(0, good));
+		if (succeed != good) {
+			return succeed;
+		}
 	}
-	_data.flush();
+	if (!part) {
-	return count;
+		return good;
+	}
+	auto storage = bytes::array<kBlockSize>();
+	const auto padded = bytes::make_span(storage);
+	const auto succeed = read(padded);
+	if (!succeed) {
+		return good;
+	}
+	Assert(succeed == kBlockSize);
+	bytes::copy(bytes.subspan(good), padded.subspan(0, part));
+	return size;
+}
+
+bool File::writeWithPadding(bytes::span bytes) {
+	const auto size = bytes.size();
+	const auto part = size % kBlockSize;
+	const auto good = size - part;
+	if (good && !write(bytes.subspan(0, good))) {
+		return false;
+	}
+	if (!part) {
+		return true;
+	}
+	auto storage = bytes::array<kBlockSize>();
+	const auto padded = bytes::make_span(storage);
+	bytes::copy(padded, bytes.subspan(good));
+	bytes::set_random(padded.subspan(part));
+	if (write(padded)) {
+		return true;
+	}
+	if (good) {
+		decryptBack(bytes.subspan(0, good));
+		_data.seek(_data.pos() - good);
+	}
+	return false;
+}
+
+bool File::flush() {
+	return _data.flush();
 }
 
 void File::close() {
 	_lock.unlock();
 	_data.close();
 	_data.setFileName(QString());
-	_offset = 0;
+	_dataSize = _encryptionOffset = 0;
 	_state = base::none;
 }
 
+int64 File::size() const {
+	return _dataSize;
+}
+
+int64 File::offset() const {
+	const auto realOffset = kSaltSize + _encryptionOffset;
+	const auto skipOffset = sizeof(BasicHeader);
+	return (realOffset >= skipOffset) ? (realOffset - skipOffset) : 0;
+}
+
+bool File::seek(int64 offset) {
+	const auto realOffset = sizeof(BasicHeader) + offset;
+	if (offset < 0 || offset > _dataSize) {
+		return false;
+	} else if (!_data.seek(FileLock::kSkipBytes + realOffset)) {
+		return false;
+	}
+	_encryptionOffset = realOffset - kSaltSize;
+	return true;
+}
+
 } // namespace Storage

Telegram/SourceFiles/storage/storage_encrypted_file.h

@@ -30,7 +30,16 @@ public:
 	Result open(const QString &path, Mode mode, const EncryptionKey &key);
 
 	size_type read(bytes::span bytes);
-	size_type write(bytes::span bytes);
+	bool write(bytes::span bytes);
+
+	size_type readWithPadding(bytes::span bytes);
+	bool writeWithPadding(bytes::span bytes);
+
+	bool flush();
+
+	int64 size() const;
+	int64 offset() const;
+	bool seek(int64 offset);
 
 	void close();
 
@@ -52,10 +61,12 @@ private:
 	size_type writePlain(bytes::const_span bytes);
 	void decrypt(bytes::span bytes);
 	void encrypt(bytes::span bytes);
+	void decryptBack(bytes::span bytes);
 
 	QFile _data;
 	FileLock _lock;
-	index_type _offset = 0;
+	int64 _encryptionOffset = 0;
+	int64 _dataSize = 0;
 
 	base::optional<CtrState> _state;
 

Telegram/SourceFiles/storage/storage_encrypted_file_tests.cpp

@@ -24,7 +24,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 
 extern int (*TestForkedMethod)();
 
-const auto key = Storage::EncryptionKey(bytes::make_vector(
+const auto Key = Storage::EncryptionKey(bytes::make_vector(
 	bytes::make_span("\
 abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\
 abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\
@@ -32,17 +32,18 @@ abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\
 abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\
 ").subspan(0, Storage::EncryptionKey::kSize)));
 
-const auto name = QString("test.file");
+const auto Name = QString("test.file");
 
-const auto test = bytes::make_span("testbytetestbyte").subspan(0, 16);
+const auto Test1 = bytes::make_span("testbytetestbyte").subspan(0, 16);
+const auto Test2 = bytes::make_span("bytetestbytetest").subspan(0, 16);
 
 struct ForkInit {
 	static int Method() {
 		Storage::File file;
 		const auto result = file.open(
-			name,
+			Name,
 			Storage::File::Mode::ReadAppend,
-			key);
+			Key);
 		if (result != Storage::File::Result::Success) {
 			return -1;
 		}
@@ -51,12 +52,11 @@ struct ForkInit {
 		const auto read = file.read(data);
 		if (read != data.size()) {
 			return -1;
-		} else if (data != bytes::make_vector(test)) {
+		} else if (data != bytes::make_vector(Test1)) {
 			return -1;
 		}
 
-		const auto written = file.write(data);
+		if (!file.write(data) || !file.flush()) {
-		if (written != data.size()) {
 			return -1;
 		}
 #ifdef _DEBUG
@@ -85,44 +85,72 @@ TEST_CASE("simple encrypted file", "[storage_encrypted_file]") {
 	SECTION("writing file") {
 		Storage::File file;
 		const auto result = file.open(
-			name,
+			Name,
 			Storage::File::Mode::Write,
-			key);
+			Key);
 		REQUIRE(result == Storage::File::Result::Success);
 
-		auto data = bytes::make_vector(test);
+		auto data = bytes::make_vector(Test1);
-		const auto written = file.write(data);
+		const auto success = file.write(data);
-		REQUIRE(written == data.size());
+		REQUIRE(success);
 	}
 	SECTION("reading and writing file") {
 		Storage::File file;
 		const auto result = file.open(
-			name,
+			Name,
 			Storage::File::Mode::ReadAppend,
-			key);
+			Key);
 		REQUIRE(result == Storage::File::Result::Success);
 
 		auto data = bytes::vector(16);
 		const auto read = file.read(data);
 		REQUIRE(read == data.size());
-		REQUIRE(data == bytes::make_vector(test));
+		REQUIRE(data == bytes::make_vector(Test1));
 
-		const auto written = file.write(data);
+		data = bytes::make_vector(Test2);
-		REQUIRE(written == data.size());
+		const auto success = file.write(data);
+		REQUIRE(success);
+	}
+	SECTION("offset and seek") {
+		Storage::File file;
+		const auto result = file.open(
+			Name,
+			Storage::File::Mode::ReadAppend,
+			Key);
+		REQUIRE(result == Storage::File::Result::Success);
+		REQUIRE(file.offset() == 0);
+
+		const auto success1 = file.seek(16);
+		REQUIRE(success1);
+		REQUIRE(file.offset() == 16);
+
+		auto data = bytes::vector(16);
+		const auto read = file.read(data);
+		REQUIRE(read == data.size());
+		REQUIRE(data == bytes::make_vector(Test2));
+		REQUIRE(file.offset() == 32);
+
+		const auto success2 = file.seek(16);
+		REQUIRE(success2);
+		REQUIRE(file.offset() == 16);
+
+		data = bytes::make_vector(Test1);
+		const auto success3 = file.write(data);
+		REQUIRE(success3);
 	}
 	SECTION("reading file") {
 		Storage::File file;
 
 		const auto result = file.open(
-			name,
+			Name,
 			Storage::File::Mode::Read,
-			key);
+			Key);
 		REQUIRE(result == Storage::File::Result::Success);
 
 		auto data = bytes::vector(32);
 		const auto read = file.read(data);
 		REQUIRE(read == data.size());
-		REQUIRE(data == bytes::concatenate(test, test));
+		REQUIRE(data == bytes::concatenate(Test1, Test1));
 	}
 }
 
@@ -130,14 +158,14 @@ TEST_CASE("two process encrypted file", "[storage_encrypted_file]") {
 	SECTION("writing file") {
 		Storage::File file;
 		const auto result = file.open(
-			name,
+			Name,
 			Storage::File::Mode::Write,
-			key);
+			Key);
 		REQUIRE(result == Storage::File::Result::Success);
 
-		auto data = bytes::make_vector(test);
+		auto data = bytes::make_vector(Test1);
-		const auto written = file.write(data);
+		const auto success = file.write(data);
-		REQUIRE(written == data.size());
+		REQUIRE(success);
 	}
 	SECTION("access from subprocess") {
 		SECTION("start subprocess") {
@@ -172,15 +200,15 @@ TEST_CASE("two process encrypted file", "[storage_encrypted_file]") {
 			Storage::File file;
 
 			const auto result = file.open(
-				name,
+				Name,
 				Storage::File::Mode::Read,
-				key);
+				Key);
 			REQUIRE(result == Storage::File::Result::Success);
 
 			auto data = bytes::vector(32);
 			const auto read = file.read(data);
 			REQUIRE(read == data.size());
-			REQUIRE(data == bytes::concatenate(test, test));
+			REQUIRE(data == bytes::concatenate(Test1, Test1));
 		}
 		SECTION("take subprocess result") {
 			REQUIRE(ForkProcess.state() == QProcess::Running);
@@ -188,15 +216,15 @@ TEST_CASE("two process encrypted file", "[storage_encrypted_file]") {
 			Storage::File file;
 
 			const auto result = file.open(
-				name,
+				Name,
 				Storage::File::Mode::ReadAppend,
-				key);
+				Key);
 			REQUIRE(result == Storage::File::Result::Success);
 
 			auto data = bytes::vector(32);
 			const auto read = file.read(data);
 			REQUIRE(read == data.size());
-			REQUIRE(data == bytes::concatenate(test, test));
+			REQUIRE(data == bytes::concatenate(Test1, Test1));
 
 			const auto finished = ForkProcess.waitForFinished(0);
 			REQUIRE(finished);

Telegram/SourceFiles/storage/storage_encryption.cpp

@@ -20,7 +20,7 @@ CtrState::CtrState(bytes::const_span key, bytes::const_span iv) {
 }
 
 template <typename Method>
-void CtrState::process(bytes::span data, index_type offset, Method method) {
+void CtrState::process(bytes::span data, int64 offset, Method method) {
 	Expects((data.size() % kBlockSize) == 0);
 	Expects((offset % kBlockSize) == 0);
 
@@ -46,7 +46,7 @@ void CtrState::process(bytes::span data, index_type offset, Method method) {
 		(block128_f)method);
 }
 
-auto CtrState::incrementedIv(index_type blockIndex)
+auto CtrState::incrementedIv(int64 blockIndex)
 -> bytes::array<kIvSize> {
 	Expects(blockIndex >= 0);
 
@@ -65,11 +65,11 @@ auto CtrState::incrementedIv(index_type blockIndex)
 	return result;
 }
 
-void CtrState::encrypt(bytes::span data, index_type offset) {
+void CtrState::encrypt(bytes::span data, int64 offset) {
 	return process(data, offset, AES_encrypt);
 }
 
-void CtrState::decrypt(bytes::span data, index_type offset) {
+void CtrState::decrypt(bytes::span data, int64 offset) {
 	return process(data, offset, AES_encrypt);
 }
 
@@ -78,6 +78,14 @@ EncryptionKey::EncryptionKey(bytes::vector &&data)
 	Expects(_data.size() == kSize);
 }
 
+bool EncryptionKey::empty() const {
+	return _data.empty();
+}
+
+EncryptionKey::operator bool() const {
+	return !empty();
+}
+
 const bytes::vector &EncryptionKey::data() const {
 	return _data;
 }

Telegram/SourceFiles/storage/storage_encryption.h

@@ -21,14 +21,14 @@ public:
 
 	CtrState(bytes::const_span key, bytes::const_span iv);
 
-	void encrypt(bytes::span data, index_type offset);
+	void encrypt(bytes::span data, int64 offset);
-	void decrypt(bytes::span data, index_type offset);
+	void decrypt(bytes::span data, int64 offset);
 
 private:
 	template <typename Method>
-	void process(bytes::span data, index_type offset, Method method);
+	void process(bytes::span data, int64 offset, Method method);
 
-	bytes::array<kIvSize> incrementedIv(index_type blockIndex);
+	bytes::array<kIvSize> incrementedIv(int64 blockIndex);
 
 	static constexpr auto EcountSize = kBlockSize;
 
@@ -44,6 +44,9 @@ public:
 	EncryptionKey() = default;
 	explicit EncryptionKey(bytes::vector &&data);
 
+	bool empty() const;
+	explicit operator bool() const;
+
 	const bytes::vector &data() const;
 	CtrState prepareCtrState(bytes::const_span salt) const;
 

Telegram/SourceFiles/storage/storage_file_lock.h

@@ -8,7 +8,6 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 #pragma once
 
 #include "core/basic_types.h"
-
 #include <QtCore/QFile>
 
 namespace Storage {

Telegram/SourceFiles/storage/storage_pch.h

@@ -16,6 +16,8 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 #include <rpl/rpl.h>
 
 #include <vector>
+#include <unordered_map>
+#include <set>
 
 #include <range/v3/all.hpp>
 #ifdef Q_OS_WIN

Telegram/SourceFiles/ui/animation.h

@@ -164,7 +164,7 @@ void startManager();
 void stopManager();
 void registerClipManager(Media::Clip::Manager *manager);
 
-FORCE_INLINE int interpolate(int a, int b, float64 b_ratio) {
+TG_FORCE_INLINE int interpolate(int a, int b, float64 b_ratio) {
 	return qRound(a + float64(b - a) * b_ratio);
 }
 
@@ -184,36 +184,36 @@ struct Shifted {
 	uint32 high = 0;
 };
 
-FORCE_INLINE Shifted operator+(Shifted a, Shifted b) {
+TG_FORCE_INLINE Shifted operator+(Shifted a, Shifted b) {
 	return Shifted(a.low + b.low, a.high + b.high);
 }
 
-FORCE_INLINE Shifted operator*(Shifted shifted, ShiftedMultiplier multiplier) {
+TG_FORCE_INLINE Shifted operator*(Shifted shifted, ShiftedMultiplier multiplier) {
 	return Shifted(shifted.low * multiplier, shifted.high * multiplier);
 }
 
-FORCE_INLINE Shifted operator*(ShiftedMultiplier multiplier, Shifted shifted) {
+TG_FORCE_INLINE Shifted operator*(ShiftedMultiplier multiplier, Shifted shifted) {
 	return Shifted(shifted.low * multiplier, shifted.high * multiplier);
 }
 
-FORCE_INLINE Shifted shifted(uint32 components) {
+TG_FORCE_INLINE Shifted shifted(uint32 components) {
 	return Shifted(
 		(components & 0x000000FFU) | ((components & 0x0000FF00U) << 8),
 		((components & 0x00FF0000U) >> 16) | ((components & 0xFF000000U) >> 8));
 }
 
-FORCE_INLINE uint32 unshifted(Shifted components) {
+TG_FORCE_INLINE uint32 unshifted(Shifted components) {
 	return ((components.low & 0x0000FF00U) >> 8)
 		| ((components.low & 0xFF000000U) >> 16)
 		| ((components.high & 0x0000FF00U) << 8)
 		| (components.high & 0xFF000000U);
 }
 
-FORCE_INLINE Shifted reshifted(Shifted components) {
+TG_FORCE_INLINE Shifted reshifted(Shifted components) {
 	return Shifted((components.low >> 8) & 0x00FF00FFU, (components.high >> 8) & 0x00FF00FFU);
 }
 
-FORCE_INLINE Shifted shifted(QColor color) {
+TG_FORCE_INLINE Shifted shifted(QColor color) {
 	// Make it premultiplied.
 	auto alpha = static_cast<uint32>((color.alpha() & 0xFF) + 1);
 	auto components = Shifted(static_cast<uint32>(color.blue() & 0xFF) | (static_cast<uint32>(color.green() & 0xFF) << 16),
@@ -221,7 +221,7 @@ FORCE_INLINE Shifted shifted(QColor color) {
 	return reshifted(components * alpha);
 }
 
-FORCE_INLINE uint32 getPremultiplied(QColor color) {
+TG_FORCE_INLINE uint32 getPremultiplied(QColor color) {
 	// Make it premultiplied.
 	auto alpha = static_cast<uint32>((color.alpha() & 0xFF) + 1);
 	auto components = Shifted(static_cast<uint32>(color.blue() & 0xFF) | (static_cast<uint32>(color.green() & 0xFF) << 16),
@@ -229,16 +229,16 @@ FORCE_INLINE uint32 getPremultiplied(QColor color) {
 	return unshifted(components * alpha);
 }
 
-FORCE_INLINE uint32 getAlpha(Shifted components) {
+TG_FORCE_INLINE uint32 getAlpha(Shifted components) {
 	return (components.high & 0x00FF0000U) >> 16;
 }
 
-FORCE_INLINE Shifted non_premultiplied(QColor color) {
+TG_FORCE_INLINE Shifted non_premultiplied(QColor color) {
 	return Shifted(static_cast<uint32>(color.blue() & 0xFF) | (static_cast<uint32>(color.green() & 0xFF) << 16),
 		static_cast<uint32>(color.red() & 0xFF) | (static_cast<uint32>(color.alpha() & 0xFF) << 16));
 }
 
-FORCE_INLINE QColor color(QColor a, QColor b, float64 b_ratio) {
+TG_FORCE_INLINE QColor color(QColor a, QColor b, float64 b_ratio) {
 	auto bOpacity = snap(interpolate(0, 255, b_ratio), 0, 255) + 1;
 	auto aOpacity = (256 - bOpacity);
 	auto components = (non_premultiplied(a) * aOpacity + non_premultiplied(b) * bOpacity);
@@ -263,19 +263,19 @@ struct Shifted {
 	uint64 value = 0;
 };
 
-FORCE_INLINE Shifted operator+(Shifted a, Shifted b) {
+TG_FORCE_INLINE Shifted operator+(Shifted a, Shifted b) {
 	return Shifted(a.value + b.value);
 }
 
-FORCE_INLINE Shifted operator*(Shifted shifted, ShiftedMultiplier multiplier) {
+TG_FORCE_INLINE Shifted operator*(Shifted shifted, ShiftedMultiplier multiplier) {
 	return Shifted(shifted.value * multiplier);
 }
 
-FORCE_INLINE Shifted operator*(ShiftedMultiplier multiplier, Shifted shifted) {
+TG_FORCE_INLINE Shifted operator*(ShiftedMultiplier multiplier, Shifted shifted) {
 	return Shifted(shifted.value * multiplier);
 }
 
-FORCE_INLINE Shifted shifted(uint32 components) {
+TG_FORCE_INLINE Shifted shifted(uint32 components) {
 	auto wide = static_cast<uint64>(components);
 	return (wide & 0x00000000000000FFULL)
 		| ((wide & 0x000000000000FF00ULL) << 8)
@@ -283,18 +283,18 @@ FORCE_INLINE Shifted shifted(uint32 components) {
 		| ((wide & 0x00000000FF000000ULL) << 24);
 }
 
-FORCE_INLINE uint32 unshifted(Shifted components) {
+TG_FORCE_INLINE uint32 unshifted(Shifted components) {
 	return static_cast<uint32>((components.value & 0x000000000000FF00ULL) >> 8)
 		| static_cast<uint32>((components.value & 0x00000000FF000000ULL) >> 16)
 		| static_cast<uint32>((components.value & 0x0000FF0000000000ULL) >> 24)
 		| static_cast<uint32>((components.value & 0xFF00000000000000ULL) >> 32);
 }
 
-FORCE_INLINE Shifted reshifted(Shifted components) {
+TG_FORCE_INLINE Shifted reshifted(Shifted components) {
 	return (components.value >> 8) & 0x00FF00FF00FF00FFULL;
 }
 
-FORCE_INLINE Shifted shifted(QColor color) {
+TG_FORCE_INLINE Shifted shifted(QColor color) {
 	// Make it premultiplied.
 	auto alpha = static_cast<uint64>((color.alpha() & 0xFF) + 1);
 	auto components = static_cast<uint64>(color.blue() & 0xFF)
@@ -304,7 +304,7 @@ FORCE_INLINE Shifted shifted(QColor color) {
 	return reshifted(components * alpha);
 }
 
-FORCE_INLINE uint32 getPremultiplied(QColor color) {
+TG_FORCE_INLINE uint32 getPremultiplied(QColor color) {
 	// Make it premultiplied.
 	auto alpha = static_cast<uint64>((color.alpha() & 0xFF) + 1);
 	auto components = static_cast<uint64>(color.blue() & 0xFF)
@@ -314,18 +314,18 @@ FORCE_INLINE uint32 getPremultiplied(QColor color) {
 	return unshifted(components * alpha);
 }
 
-FORCE_INLINE uint32 getAlpha(Shifted components) {
+TG_FORCE_INLINE uint32 getAlpha(Shifted components) {
 	return (components.value & 0x00FF000000000000ULL) >> 48;
 }
 
-FORCE_INLINE Shifted non_premultiplied(QColor color) {
+TG_FORCE_INLINE Shifted non_premultiplied(QColor color) {
 	return static_cast<uint64>(color.blue() & 0xFF)
 		| (static_cast<uint64>(color.green() & 0xFF) << 16)
 		| (static_cast<uint64>(color.red() & 0xFF) << 32)
 		| (static_cast<uint64>(color.alpha() & 0xFF) << 48);
 }
 
-FORCE_INLINE QColor color(QColor a, QColor b, float64 b_ratio) {
+TG_FORCE_INLINE QColor color(QColor a, QColor b, float64 b_ratio) {
 	auto bOpacity = snap(interpolate(0, 255, b_ratio), 0, 255) + 1;
 	auto aOpacity = (256 - bOpacity);
 	auto components = (non_premultiplied(a) * aOpacity + non_premultiplied(b) * bOpacity);
@@ -339,47 +339,47 @@ FORCE_INLINE QColor color(QColor a, QColor b, float64 b_ratio) {
 
 #endif // SHIFTED_USE_32BIT
 
-FORCE_INLINE QColor color(style::color a, QColor b, float64 b_ratio) {
+TG_FORCE_INLINE QColor color(style::color a, QColor b, float64 b_ratio) {
 	return color(a->c, b, b_ratio);
 }
 
-FORCE_INLINE QColor color(QColor a, style::color b, float64 b_ratio) {
+TG_FORCE_INLINE QColor color(QColor a, style::color b, float64 b_ratio) {
 	return color(a, b->c, b_ratio);
 }
 
-FORCE_INLINE QColor color(style::color a, style::color b, float64 b_ratio) {
+TG_FORCE_INLINE QColor color(style::color a, style::color b, float64 b_ratio) {
 	return color(a->c, b->c, b_ratio);
 }
 
-FORCE_INLINE QPen pen(QColor a, QColor b, float64 b_ratio) {
+TG_FORCE_INLINE QPen pen(QColor a, QColor b, float64 b_ratio) {
 	return color(a, b, b_ratio);
 }
 
-FORCE_INLINE QPen pen(style::color a, QColor b, float64 b_ratio) {
+TG_FORCE_INLINE QPen pen(style::color a, QColor b, float64 b_ratio) {
 	return (b_ratio > 0) ? pen(a->c, b, b_ratio) : a;
 }
 
-FORCE_INLINE QPen pen(QColor a, style::color b, float64 b_ratio) {
+TG_FORCE_INLINE QPen pen(QColor a, style::color b, float64 b_ratio) {
 	return (b_ratio < 1) ? pen(a, b->c, b_ratio) : b;
 }
 
-FORCE_INLINE QPen pen(style::color a, style::color b, float64 b_ratio) {
+TG_FORCE_INLINE QPen pen(style::color a, style::color b, float64 b_ratio) {
 	return (b_ratio > 0) ? ((b_ratio < 1) ? pen(a->c, b->c, b_ratio) : b) : a;
 }
 
-FORCE_INLINE QBrush brush(QColor a, QColor b, float64 b_ratio) {
+TG_FORCE_INLINE QBrush brush(QColor a, QColor b, float64 b_ratio) {
 	return color(a, b, b_ratio);
 }
 
-FORCE_INLINE QBrush brush(style::color a, QColor b, float64 b_ratio) {
+TG_FORCE_INLINE QBrush brush(style::color a, QColor b, float64 b_ratio) {
 	return (b_ratio > 0) ? brush(a->c, b, b_ratio) : a;
 }
 
-FORCE_INLINE QBrush brush(QColor a, style::color b, float64 b_ratio) {
+TG_FORCE_INLINE QBrush brush(QColor a, style::color b, float64 b_ratio) {
 	return (b_ratio < 1) ? brush(a, b->c, b_ratio) : b;
 }
 
-FORCE_INLINE QBrush brush(style::color a, style::color b, float64 b_ratio) {
+TG_FORCE_INLINE QBrush brush(style::color a, style::color b, float64 b_ratio) {
 	return (b_ratio > 0) ? ((b_ratio < 1) ? brush(a->c, b->c, b_ratio) : b) : a;
 }
 

Telegram/SourceFiles/ui/images.cpp

@@ -17,7 +17,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 namespace Images {
 namespace {
 
-FORCE_INLINE uint64 blurGetColors(const uchar *p) {
+TG_FORCE_INLINE uint64 blurGetColors(const uchar *p) {
 	return (uint64)p[0] + ((uint64)p[1] << 16) + ((uint64)p[2] << 32) + ((uint64)p[3] << 48);
 }
 

Telegram/gyp/Telegram.gyp

@@ -61,7 +61,7 @@
       'telegram_win.gypi',
       'telegram_mac.gypi',
       'telegram_linux.gypi',
-	  'openssl.gypi',
+      'openssl.gypi',
       'qt.gypi',
       'qt_moc.gypi',
       'qt_rcc.gypi',
@@ -86,6 +86,7 @@
       'AL_LIBTYPE_STATIC',
       'AL_ALEXT_PROTOTYPES',
       'TGVOIP_USE_CXX11_LIB',
+      'XXH_INLINE_ALL',
       '<!@(python -c "for s in \'<(build_defines)\'.split(\',\'): print(s)")',
     ],
 
@@ -105,6 +106,7 @@
       '<(submodules_loc)/GSL/include',
       '<(submodules_loc)/variant/include',
       '<(submodules_loc)/crl/src',
+      '<(submodules_loc)/xxHash',
     ],
     'sources': [
       '<@(qrc_files)',

Telegram/gyp/common.gypi

@@ -93,6 +93,9 @@
 
   },
 
+  'defines': [
+    'NOMINMAX'
+  ],
   'configurations': {
     'Debug': {
       'defines': [

Telegram/gyp/lib_storage.gyp

@@ -30,6 +30,7 @@
       'pch_header': '<(src_loc)/storage/storage_pch.h',
     },
     'defines': [
+      'XXH_INLINE_ALL',
     ],
     'dependencies': [
       'crl.gyp:crl',
@@ -41,6 +42,7 @@
       '<(submodules_loc)/GSL/include',
       '<(submodules_loc)/variant/include',
       '<(submodules_loc)/crl/src',
+      '<(submodules_loc)/xxHash',
     ],
     'sources': [
       '<(src_loc)/storage/storage_encryption.cpp',

Telegram/gyp/tests/common_test.gypi

@@ -14,6 +14,7 @@
     '<(submodules_loc)/GSL/include',
     '<(submodules_loc)/variant/include',
     '<(submodules_loc)/Catch/include',
+    '<(submodules_loc)/crl/src',
     '<(libs_loc)/range-v3/include',
   ],
   'sources': [

Telegram/gyp/tests/tests.gyp

@@ -125,17 +125,19 @@
     ],
     'dependencies': [
       '../lib_storage.gyp:lib_storage',
+      '../crl.gyp:crl',
     ],
     'sources': [
       '<(src_loc)/storage/storage_encrypted_file_tests.cpp',
+      '<(src_loc)/storage/cache/storage_cache_database_tests.cpp',
       '<(src_loc)/platform/win/windows_dlls.cpp',
       '<(src_loc)/platform/win/windows_dlls.h',
     ],
     'conditions': [[ 'not build_win', {
-	  'sources!': [
+      'sources!': [
         '<(src_loc)/platform/win/windows_dlls.cpp',
         '<(src_loc)/platform/win/windows_dlls.h',
-	  ],
+      ],
-	}]],
+    }]],
   }],
 }