From 2d7f6fc2e7ba7a432a0ae30d706309aa0a25ca7e Mon Sep 17 00:00:00 2001 From: John Preston Date: Tue, 21 Jan 2020 14:48:17 +0300 Subject: [PATCH] Removed old lib_storage sources. --- .../cache/storage_cache_binlog_reader.cpp | 111 -- .../cache/storage_cache_binlog_reader.h | 265 ---- .../storage/cache/storage_cache_cleaner.cpp | 111 -- .../storage/cache/storage_cache_cleaner.h | 36 - .../storage/cache/storage_cache_compactor.cpp | 434 ------ .../storage/cache/storage_cache_compactor.h | 55 - .../storage/cache/storage_cache_database.cpp | 203 --- .../storage/cache/storage_cache_database.h | 91 -- .../cache/storage_cache_database_object.cpp | 1328 ----------------- .../cache/storage_cache_database_object.h | 256 ---- .../cache/storage_cache_database_tests.cpp | 740 --------- .../storage/cache/storage_cache_types.cpp | 138 -- .../storage/cache/storage_cache_types.h | 239 --- .../storage/storage_clear_legacy.cpp | 53 - .../storage/storage_clear_legacy.h | 26 - .../storage/storage_clear_legacy_posix.cpp | 101 -- .../storage/storage_clear_legacy_win.cpp | 62 - .../SourceFiles/storage/storage_databases.cpp | 104 -- .../SourceFiles/storage/storage_databases.h | 71 - .../storage/storage_encrypted_file.cpp | 348 ----- .../storage/storage_encrypted_file.h | 73 - .../storage/storage_encrypted_file_tests.cpp | 247 --- .../storage/storage_encryption.cpp | 109 -- .../SourceFiles/storage/storage_encryption.h | 58 - Telegram/SourceFiles/storage/storage_pch.cpp | 10 - Telegram/SourceFiles/storage/storage_pch.h | 30 - 26 files changed, 5299 deletions(-) delete mode 100644 Telegram/SourceFiles/storage/cache/storage_cache_binlog_reader.cpp delete mode 100644 Telegram/SourceFiles/storage/cache/storage_cache_binlog_reader.h delete mode 100644 Telegram/SourceFiles/storage/cache/storage_cache_cleaner.cpp delete mode 100644 Telegram/SourceFiles/storage/cache/storage_cache_cleaner.h delete mode 100644 Telegram/SourceFiles/storage/cache/storage_cache_compactor.cpp delete mode 100644 Telegram/SourceFiles/storage/cache/storage_cache_compactor.h delete mode 100644 Telegram/SourceFiles/storage/cache/storage_cache_database.cpp delete mode 100644 Telegram/SourceFiles/storage/cache/storage_cache_database.h delete mode 100644 Telegram/SourceFiles/storage/cache/storage_cache_database_object.cpp delete mode 100644 Telegram/SourceFiles/storage/cache/storage_cache_database_object.h delete mode 100644 Telegram/SourceFiles/storage/cache/storage_cache_database_tests.cpp delete mode 100644 Telegram/SourceFiles/storage/cache/storage_cache_types.cpp delete mode 100644 Telegram/SourceFiles/storage/cache/storage_cache_types.h delete mode 100644 Telegram/SourceFiles/storage/storage_clear_legacy.cpp delete mode 100644 Telegram/SourceFiles/storage/storage_clear_legacy.h delete mode 100644 Telegram/SourceFiles/storage/storage_clear_legacy_posix.cpp delete mode 100644 Telegram/SourceFiles/storage/storage_clear_legacy_win.cpp delete mode 100644 Telegram/SourceFiles/storage/storage_databases.cpp delete mode 100644 Telegram/SourceFiles/storage/storage_databases.h delete mode 100644 Telegram/SourceFiles/storage/storage_encrypted_file.cpp delete mode 100644 Telegram/SourceFiles/storage/storage_encrypted_file.h delete mode 100644 Telegram/SourceFiles/storage/storage_encrypted_file_tests.cpp delete mode 100644 Telegram/SourceFiles/storage/storage_encryption.cpp delete mode 100644 Telegram/SourceFiles/storage/storage_encryption.h delete mode 100644 Telegram/SourceFiles/storage/storage_pch.cpp delete mode 100644 Telegram/SourceFiles/storage/storage_pch.h diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_binlog_reader.cpp b/Telegram/SourceFiles/storage/cache/storage_cache_binlog_reader.cpp deleted file mode 100644 index 432aa0d5e..000000000 --- a/Telegram/SourceFiles/storage/cache/storage_cache_binlog_reader.cpp +++ /dev/null @@ -1,111 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#include "storage/cache/storage_cache_binlog_reader.h" - -namespace Storage { -namespace Cache { -namespace details { - -BinlogWrapper::BinlogWrapper( - File &binlog, - const Settings &settings, - int64 till) -: _binlog(binlog) -, _settings(settings) -, _till(till ? till : _binlog.size()) -, _data(_settings.readBlockSize) -, _full(_data) { -} - -bool BinlogWrapper::finished() const { - return _finished; -} - -bool BinlogWrapper::failed() const { - return _failed; -} - -std::optional BinlogWrapper::ReadHeader( - File &binlog, - const Settings &settings) { - auto result = BasicHeader(); - if (binlog.offset() != 0) { - return {}; - } else if (binlog.read(bytes::object_as_span(&result)) != sizeof(result)) { - return {}; - } else if (result.getFormat() != Format::Format_0) { - return {}; - } else if (settings.trackEstimatedTime - != !!(result.flags & result.kTrackEstimatedTime)) { - return {}; - } - return result; -} - -bool BinlogWrapper::readPart() { - if (_finished) { - return false; - } - const auto no = [&] { - finish(); - return false; - }; - const auto offset = _binlog.offset(); - const auto left = (_till - offset); - if (!left) { - return no(); - } - - if (!_part.empty() && _full.data() != _part.data()) { - bytes::move(_full, _part); - _part = _full.subspan(0, _part.size()); - } - const auto amount = std::min( - left, - int64(_full.size() - _part.size())); - Assert(amount > 0); - const auto readBytes = _binlog.read( - _full.subspan(_part.size(), amount)); - if (!readBytes) { - return no(); - } - _part = _full.subspan(0, _part.size() + readBytes); - return true; -} - -bytes::const_span BinlogWrapper::readRecord(ReadRecordSize readRecordSize) { - if (_finished) { - return {}; - } - const auto size = readRecordSize(*this, _part); - if (size == kRecordSizeUnknown || size > _part.size()) { - return {}; - } else if (size == kRecordSizeInvalid) { - finish(); - _finished = _failed = true; - return {}; - } - Assert(size >= 0); - const auto result = _part.subspan(0, size); - _part = _part.subspan(size); - return result; -} - -void BinlogWrapper::finish(size_type rollback) { - Expects(rollback >= 0); - - if (rollback > 0) { - _failed = true; - } - rollback += _part.size(); - _binlog.seek(_binlog.offset() - rollback); -} - -} // namespace details -} // namespace Cache -} // namespace Storage diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_binlog_reader.h b/Telegram/SourceFiles/storage/cache/storage_cache_binlog_reader.h deleted file mode 100644 index 379f5742d..000000000 --- a/Telegram/SourceFiles/storage/cache/storage_cache_binlog_reader.h +++ /dev/null @@ -1,265 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#pragma once - -#include "storage/cache/storage_cache_types.h" -#include "storage/storage_encrypted_file.h" -#include "base/bytes.h" -#include "base/match_method.h" - -namespace Storage { -namespace Cache { -namespace details { - -template -class BinlogReader; - -class BinlogWrapper { -public: - BinlogWrapper(File &binlog, const Settings &settings, int64 till = 0); - - bool finished() const; - bool failed() const; - - static std::optional ReadHeader( - File &binlog, - const Settings &settings); - -private: - template - friend class BinlogReader; - - bool readPart(); - void finish(size_type rollback = 0); - - using ReadRecordSize = size_type (*)( - const BinlogWrapper &that, - bytes::const_span data); - bytes::const_span readRecord(ReadRecordSize readRecordSize); - - File &_binlog; - Settings _settings; - - int64 _till = 0; - bytes::vector _data; - bytes::span _full; - bytes::span _part; - bool _finished = false; - bool _failed = false; - -}; - -template -class BinlogReader { -public: - explicit BinlogReader(BinlogWrapper &wrapper); - - template - bool readTillEnd(Handlers &&...handlers); - -private: - static size_type ReadRecordSize( - const BinlogWrapper &that, - bytes::const_span data); - - template - bool handleRecord(bytes::const_span data, Handlers &&...handlers) const; - - BinlogWrapper &_wrapper; - -}; - -template -struct MultiRecord { - using true_t = char; - using false_t = true_t(&)[2]; - static_assert(sizeof(true_t) != sizeof(false_t)); - - static false_t Check(...); - template - static true_t Check(const Test&); - - static constexpr bool Is = (sizeof(Check(std::declval())) - == sizeof(true_t)); -}; - -template -struct BinlogReaderRecursive { - static void CheckSettings(const Settings &settings) { - } - - static size_type ReadRecordSize( - RecordType type, - bytes::const_span data, - size_type partsLimit) { - return kRecordSizeInvalid; - } - - template - static bool HandleRecord( - RecordType type, - bytes::const_span data, - Handlers &&...handlers) { - Unexpected("Bad type in BinlogReaderRecursive::HandleRecord."); - } -}; - -template -struct BinlogReaderRecursive { - static void CheckSettings(const Settings &settings); - - static size_type ReadRecordSize( - RecordType type, - bytes::const_span data, - size_type partsLimit); - - template - static bool HandleRecord( - RecordType type, - bytes::const_span data, - Handlers &&...handlers); -}; - -template -inline void BinlogReaderRecursive::CheckSettings( - const Settings &settings) { - static_assert(GoodForEncryption); - if constexpr (MultiRecord::Is) { - using Head = Record; - using Part = typename Record::Part; - static_assert(GoodForEncryption); - Assert(settings.readBlockSize - >= (sizeof(Head) - + settings.maxBundledRecords * sizeof(Part))); - } else { - Assert(settings.readBlockSize >= sizeof(Record)); - } -} - -template -inline size_type BinlogReaderRecursive::ReadRecordSize( - RecordType type, - bytes::const_span data, - size_type partsLimit) { - if (type != Record::kType) { - return BinlogReaderRecursive::ReadRecordSize( - type, - data, - partsLimit); - } - if constexpr (MultiRecord::Is) { - using Head = Record; - using Part = typename Record::Part; - - if (data.size() < sizeof(Head)) { - return kRecordSizeUnknown; - } - const auto head = reinterpret_cast(data.data()); - const auto count = head->validateCount(); - return (count >= 0 && count <= partsLimit) - ? (sizeof(Head) + count * sizeof(Part)) - : kRecordSizeInvalid; - } else { - return sizeof(Record); - } -} - -template -template -inline bool BinlogReaderRecursive::HandleRecord( - RecordType type, - bytes::const_span data, - Handlers &&...handlers) { - if (type != Record::kType) { - return BinlogReaderRecursive::HandleRecord( - type, - data, - std::forward(handlers)...); - } - if constexpr (MultiRecord::Is) { - using Head = Record; - using Part = typename Record::Part; - - Assert(data.size() >= sizeof(Head)); - const auto bytes = data.data(); - const auto head = reinterpret_cast(bytes); - const auto count = head->validateCount(); - Assert(data.size() == sizeof(Head) + count * sizeof(Part)); - const auto parts = gsl::make_span( - reinterpret_cast(bytes + sizeof(Head)), - count); - auto from = std::begin(parts); - const auto till = std::end(parts); - const auto element = [&] { - return (from == till) ? nullptr : &*from++; - }; - return base::match_method2( - *head, - element, - std::forward(handlers)...); - } else { - Assert(data.size() == sizeof(Record)); - return base::match_method( - *reinterpret_cast(data.data()), - std::forward(handlers)...); - } -} - -template -BinlogReader::BinlogReader(BinlogWrapper &wrapper) -: _wrapper(wrapper) { - BinlogReaderRecursive::CheckSettings(_wrapper._settings); -} - -template -template -bool BinlogReader::readTillEnd(Handlers &&...handlers) { - if (!_wrapper.readPart()) { - return true; - } - const auto readRecord = [&] { - return _wrapper.readRecord(&BinlogReader::ReadRecordSize); - }; - for (auto bytes = readRecord(); !bytes.empty(); bytes = readRecord()) { - if (!handleRecord(bytes, std::forward(handlers)...)) { - _wrapper.finish(bytes.size()); - return true; - } - } - return false; -} - -template -size_type BinlogReader::ReadRecordSize( - const BinlogWrapper &that, - bytes::const_span data) { - if (data.empty()) { - return kRecordSizeUnknown; - } - return BinlogReaderRecursive::ReadRecordSize( - static_cast(data[0]), - data, - that._settings.maxBundledRecords); -} - -template -template -bool BinlogReader::handleRecord( - bytes::const_span data, - Handlers &&...handlers) const { - Expects(!data.empty()); - - return BinlogReaderRecursive::HandleRecord( - static_cast(data[0]), - data, - std::forward(handlers)...); -} - -} // namespace details -} // namespace Cache -} // namespace Storage diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_cleaner.cpp b/Telegram/SourceFiles/storage/cache/storage_cache_cleaner.cpp deleted file mode 100644 index 7e6a18d0d..000000000 --- a/Telegram/SourceFiles/storage/cache/storage_cache_cleaner.cpp +++ /dev/null @@ -1,111 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#include "storage/cache/storage_cache_cleaner.h" - -#include -#include -#include -#include -#include - -namespace Storage { -namespace Cache { -namespace details { - -class CleanerObject { -public: - CleanerObject( - crl::weak_on_queue weak, - const QString &base, - base::binary_guard &&guard, - FnMut done); - -private: - void start(); - void scheduleNext(); - void cleanNext(); - void done(); - - crl::weak_on_queue _weak; - QString _base, _errorPath; - std::vector _queue; - base::binary_guard _guard; - FnMut _done; - -}; - -CleanerObject::CleanerObject( - crl::weak_on_queue weak, - const QString &base, - base::binary_guard &&guard, - FnMut done) -: _weak(std::move(weak)) -, _base(base) -, _guard(std::move(guard)) -, _done(std::move(done)) { - start(); -} - -void CleanerObject::start() { - const auto entries = QDir(_base).entryList( - QDir::Dirs | QDir::NoDotAndDotDot); - for (const auto entry : entries) { - _queue.push_back(entry); - } - if (const auto version = ReadVersionValue(_base)) { - _queue.erase( - ranges::remove(_queue, QString::number(*version)), - end(_queue)); - scheduleNext(); - } else { - _errorPath = VersionFilePath(_base); - done(); - } -} - -void CleanerObject::scheduleNext() { - if (_queue.empty()) { - done(); - return; - } - _weak.with([](CleanerObject &that) { - if (that._guard) { - that.cleanNext(); - } - }); -} - -void CleanerObject::cleanNext() { - const auto path = _base + _queue.back(); - _queue.pop_back(); - if (!QDir(path).removeRecursively()) { - _errorPath = path; - } - scheduleNext(); -} - -void CleanerObject::done() { - if (_done) { - _done(_errorPath.isEmpty() - ? Error::NoError() - : Error{ Error::Type::IO, _errorPath }); - } -} - -Cleaner::Cleaner( - const QString &base, - base::binary_guard &&guard, - FnMut done) -: _wrapped(base, std::move(guard), std::move(done)) { -} - -Cleaner::~Cleaner() = default; - -} // namespace details -} // namespace Cache -} // namespace Storage diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_cleaner.h b/Telegram/SourceFiles/storage/cache/storage_cache_cleaner.h deleted file mode 100644 index 26f6f28b8..000000000 --- a/Telegram/SourceFiles/storage/cache/storage_cache_cleaner.h +++ /dev/null @@ -1,36 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#pragma once - -#include "storage/cache/storage_cache_types.h" -#include "base/binary_guard.h" - -namespace Storage { -namespace Cache { -namespace details { - -class CleanerObject; - -class Cleaner { -public: - Cleaner( - const QString &base, - base::binary_guard &&guard, - FnMut done); - - ~Cleaner(); - -private: - using Implementation = details::CleanerObject; - crl::object_on_queue _wrapped; - -}; - -} // namespace details -} // namespace Cache -} // namespace Storage diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_compactor.cpp b/Telegram/SourceFiles/storage/cache/storage_cache_compactor.cpp deleted file mode 100644 index 10f49fbeb..000000000 --- a/Telegram/SourceFiles/storage/cache/storage_cache_compactor.cpp +++ /dev/null @@ -1,434 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#include "storage/cache/storage_cache_compactor.h" - -#include "storage/cache/storage_cache_database_object.h" -#include "storage/cache/storage_cache_binlog_reader.h" -#include - -namespace Storage { -namespace Cache { -namespace details { - -class CompactorObject { -public: - using Info = Compactor::Info; - - CompactorObject( - crl::weak_on_queue weak, - crl::weak_on_queue database, - base::binary_guard guard, - const QString &base, - const Settings &settings, - EncryptionKey &&key, - const Info &info); - -private: - using Entry = DatabaseObject::Entry; - using Raw = DatabaseObject::Raw; - using RawSpan = gsl::span; - static QString CompactFilename(); - - void start(); - QString binlogPath() const; - QString compactPath() const; - bool openBinlog(); - bool readHeader(); - bool openCompact(); - void parseChunk(); - void fail(); - void done(int64 till); - void finish(); - void finalize(); - - std::vector readChunk(); - bool readBlock(std::vector &result); - void processValues(const std::vector &values); - - template - void initList(); - RawSpan fillList(RawSpan values); - template - RawSpan fillList(std::vector &list, RawSpan values); - template - void addListRecord( - std::vector &list, - const Raw &raw); - bool writeList(); - template - bool writeMultiStore(); - - crl::weak_on_queue _weak; - crl::weak_on_queue _database; - base::binary_guard _guard; - QString _base; - Settings _settings; - EncryptionKey _key; - BasicHeader _header; - Info _info; - File _binlog; - File _compact; - BinlogWrapper _wrapper; - size_type _partSize = 0; - std::unordered_set _written; - base::variant< - std::vector, - std::vector> _list; - -}; - -CompactorObject::CompactorObject( - crl::weak_on_queue weak, - crl::weak_on_queue database, - base::binary_guard guard, - const QString &base, - const Settings &settings, - EncryptionKey &&key, - const Info &info) -: _weak(std::move(weak)) -, _database(std::move(database)) -, _guard(std::move(guard)) -, _base(base) -, _settings(settings) -, _key(std::move(key)) -, _info(info) -, _wrapper(_binlog, _settings, _info.till) -, _partSize(_settings.maxBundledRecords) { // Perhaps a better estimate? - Expects(_settings.compactChunkSize > 0); - - _written.reserve(_info.keysCount); - start(); -} - -template -void CompactorObject::initList() { - using Part = typename MultiRecord::Part; - auto list = std::vector(); - list.reserve(_partSize); - _list = std::move(list); -} - -void CompactorObject::start() { - if (!openBinlog() || !readHeader() || !openCompact()) { - fail(); - } - if (_settings.trackEstimatedTime) { - initList(); - } else { - initList(); - } - parseChunk(); -} - -QString CompactorObject::CompactFilename() { - return QStringLiteral("binlog-temp"); -} - -QString CompactorObject::binlogPath() const { - return _base + DatabaseObject::BinlogFilename(); -} - -QString CompactorObject::compactPath() const { - return _base + CompactFilename(); -} - -bool CompactorObject::openBinlog() { - const auto path = binlogPath(); - const auto result = _binlog.open(path, File::Mode::Read, _key); - return (result == File::Result::Success) - && (_binlog.size() >= _info.till); -} - -bool CompactorObject::readHeader() { - const auto header = BinlogWrapper::ReadHeader(_binlog, _settings); - if (!header) { - return false; - } - _header = *header; - return true; -} - -bool CompactorObject::openCompact() { - const auto path = compactPath(); - const auto result = _compact.open(path, File::Mode::Write, _key); - if (result != File::Result::Success) { - return false; - } else if (!_compact.write(bytes::object_as_span(&_header))) { - return false; - } - return true; -} - -void CompactorObject::fail() { - _compact.close(); - QFile(compactPath()).remove(); - _database.with([](DatabaseObject &database) { - database.compactorFail(); - }); -} - -void CompactorObject::done(int64 till) { - const auto path = compactPath(); - _database.with([=, good = std::move(_guard)](DatabaseObject &database) { - if (good) { - database.compactorDone(path, till); - } - }); -} - -void CompactorObject::finish() { - if (writeList()) { - finalize(); - } else { - fail(); - } -} - -void CompactorObject::finalize() { - _binlog.close(); - _compact.close(); - - auto lastCatchUp = 0; - auto from = _info.till; - while (true) { - const auto till = CatchUp( - compactPath(), - binlogPath(), - _key, - from, - _settings.readBlockSize); - if (!till) { - fail(); - return; - } else if (till == from - || (lastCatchUp > 0 && (till - from) >= lastCatchUp)) { - done(till); - return; - } - lastCatchUp = (till - from); - from = till; - } -} - -bool CompactorObject::writeList() { - if (_list.is>()) { - return writeMultiStore(); - } else if (_list.is>()) { - return writeMultiStore(); - } else { - Unexpected("List type in CompactorObject::writeList."); - } -} - -template -bool CompactorObject::writeMultiStore() { - using Part = typename MultiRecord::Part; - Assert(_list.is>()); - auto &list = _list.get_unchecked>(); - if (list.empty()) { - return true; - } - const auto guard = gsl::finally([&] { list.clear(); }); - const auto size = list.size(); - auto header = MultiRecord(size); - if (_compact.write(bytes::object_as_span(&header)) - && _compact.write(bytes::make_span(list))) { - _compact.flush(); - return true; - } - return false; -} - -std::vector CompactorObject::readChunk() { - const auto limit = _settings.compactChunkSize; - auto result = std::vector(); - while (result.size() < limit) { - if (!readBlock(result)) { - break; - } - } - return result; -} - -bool CompactorObject::readBlock(std::vector &result) { - const auto push = [&](const Store &store) { - result.push_back(store.key); - return true; - }; - const auto pushMulti = [&](const auto &element) { - while (const auto record = element()) { - push(*record); - } - return true; - }; - if (_settings.trackEstimatedTime) { - BinlogReader< - StoreWithTime, - MultiStoreWithTime, - MultiRemove, - MultiAccess> reader(_wrapper); - return !reader.readTillEnd([&](const StoreWithTime &record) { - return push(record); - }, [&](const MultiStoreWithTime &header, const auto &element) { - return pushMulti(element); - }, [&](const MultiRemove &header, const auto &element) { - return true; - }, [&](const MultiAccess &header, const auto &element) { - return true; - }); - } else { - BinlogReader< - Store, - MultiStore, - MultiRemove> reader(_wrapper); - return !reader.readTillEnd([&](const Store &record) { - return push(record); - }, [&](const MultiStore &header, const auto &element) { - return pushMulti(element); - }, [&](const MultiRemove &header, const auto &element) { - return true; - }); - } -} - -void CompactorObject::parseChunk() { - auto keys = readChunk(); - if (_wrapper.failed()) { - fail(); - return; - } else if (keys.empty()) { - finish(); - return; - } - _database.with([ - weak = _weak, - keys = std::move(keys) - ](DatabaseObject &database) { - auto result = database.getManyRaw(keys); - weak.with([result = std::move(result)](CompactorObject &that) { - that.processValues(result); - }); - }); -} - -void CompactorObject::processValues( - const std::vector> &values) { - auto left = gsl::make_span(values); - while (true) { - left = fillList(left); - if (left.empty()) { - break; - } else if (!writeList()) { - fail(); - return; - } - } - parseChunk(); -} - -auto CompactorObject::fillList(RawSpan values) -> RawSpan { - return _list.match([&](auto &list) { - return fillList(list, values); - }); -} - -template -auto CompactorObject::fillList( - std::vector &list, - RawSpan values -) -> RawSpan { - const auto b = std::begin(values); - const auto e = std::end(values); - auto i = b; - while (i != e && list.size() != _partSize) { - addListRecord(list, *i++); - } - return values.subspan(i - b); -} - -template -void CompactorObject::addListRecord( - std::vector &list, - const Raw &raw) { - if (!_written.emplace(raw.first).second) { - return; - } - auto record = RecordStore(); - record.key = raw.first; - record.setSize(raw.second.size); - record.checksum = raw.second.checksum; - record.tag = raw.second.tag; - record.place = raw.second.place; - if constexpr (std::is_same_v) { - record.time.setRelative(raw.second.useTime); - record.time.system = _info.systemTime; - } - list.push_back(record); -} - -Compactor::Compactor( - crl::weak_on_queue database, - base::binary_guard guard, - const QString &base, - const Settings &settings, - EncryptionKey &&key, - const Info &info) -: _wrapped( - std::move(database), - std::move(guard), - base, - settings, - std::move(key), - info) { -} - -Compactor::~Compactor() = default; - -int64 CatchUp( - const QString &compactPath, - const QString &binlogPath, - const EncryptionKey &key, - int64 from, - size_type block) { - File binlog, compact; - const auto result1 = binlog.open(binlogPath, File::Mode::Read, key); - if (result1 != File::Result::Success) { - return 0; - } - const auto till = binlog.size(); - if (till == from) { - return till; - } else if (till < from || !binlog.seek(from)) { - return 0; - } - const auto result2 = compact.open( - compactPath, - File::Mode::ReadAppend, - key); - if (result2 != File::Result::Success || !compact.seek(compact.size())) { - return 0; - } - auto buffer = bytes::vector(block); - auto bytes = bytes::make_span(buffer); - do { - const auto left = (till - from); - const auto limit = std::min(size_type(left), block); - const auto read = binlog.read(bytes.subspan(0, limit)); - if (!read || read > limit) { - return 0; - } else if (!compact.write(bytes.subspan(0, read))) { - return 0; - } - from += read; - } while (from != till); - return till; -} - -} // namespace details -} // namespace Cache -} // namespace Storage diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_compactor.h b/Telegram/SourceFiles/storage/cache/storage_cache_compactor.h deleted file mode 100644 index 6b1c0a84d..000000000 --- a/Telegram/SourceFiles/storage/cache/storage_cache_compactor.h +++ /dev/null @@ -1,55 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#pragma once - -#include "storage/cache/storage_cache_types.h" -#include -#include - -namespace Storage { -class EncryptionKey; -namespace Cache { -namespace details { - -class CompactorObject; -class DatabaseObject; - -class Compactor { -public: - struct Info { - int64 till = 0; - uint32 systemTime = 0; - size_type keysCount = 0; - }; - - Compactor( - crl::weak_on_queue database, - base::binary_guard guard, - const QString &base, - const Settings &settings, - EncryptionKey &&key, - const Info &info); - - ~Compactor(); - -private: - using Implementation = CompactorObject; - crl::object_on_queue _wrapped; - -}; - -int64 CatchUp( - const QString &compactPath, - const QString &binlogPath, - const EncryptionKey &key, - int64 from, - size_type block); - -} // namespace details -} // namespace Cache -} // namespace Storage diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_database.cpp b/Telegram/SourceFiles/storage/cache/storage_cache_database.cpp deleted file mode 100644 index dfe5d0454..000000000 --- a/Telegram/SourceFiles/storage/cache/storage_cache_database.cpp +++ /dev/null @@ -1,203 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#include "storage/cache/storage_cache_database.h" - -#include "storage/cache/storage_cache_database_object.h" - -namespace Storage { -namespace Cache { - -Database::Database(const QString &path, const Settings &settings) -: _wrapped(path, settings) { -} - -void Database::reconfigure(const Settings &settings) { - _wrapped.with([settings](Implementation &unwrapped) mutable { - unwrapped.reconfigure(settings); - }); -} - -void Database::updateSettings(const SettingsUpdate &update) { - _wrapped.with([update](Implementation &unwrapped) mutable { - unwrapped.updateSettings(update); - }); -} - -void Database::open(EncryptionKey &&key, FnMut &&done) { - _wrapped.with([ - key = std::move(key), - done = std::move(done) - ](Implementation &unwrapped) mutable { - unwrapped.open(std::move(key), std::move(done)); - }); -} - -void Database::close(FnMut &&done) { - _wrapped.with([ - done = std::move(done) - ](Implementation &unwrapped) mutable { - unwrapped.close(std::move(done)); - }); -} - -void Database::waitForCleaner(FnMut &&done) { - _wrapped.with([ - done = std::move(done) - ](Implementation &unwrapped) mutable { - unwrapped.waitForCleaner(std::move(done)); - }); -} - -void Database::put( - const Key &key, - QByteArray &&value, - FnMut &&done) { - return put(key, TaggedValue(std::move(value), 0), std::move(done)); -} - -void Database::get(const Key &key, FnMut &&done) { - if (done) { - auto untag = [done = std::move(done)](TaggedValue &&value) mutable { - done(std::move(value.bytes)); - }; - getWithTag(key, std::move(untag)); - } else { - getWithTag(key, nullptr); - } -} - -void Database::remove(const Key &key, FnMut &&done) { - _wrapped.with([ - key, - done = std::move(done) - ](Implementation &unwrapped) mutable { - unwrapped.remove(key, std::move(done)); - }); -} - -void Database::putIfEmpty( - const Key &key, - QByteArray &&value, - FnMut &&done) { - return putIfEmpty( - key, - TaggedValue(std::move(value), 0), - std::move(done)); -} - -void Database::copyIfEmpty( - const Key &from, - const Key &to, - FnMut &&done) { - _wrapped.with([ - from, - to, - done = std::move(done) - ](Implementation &unwrapped) mutable { - unwrapped.copyIfEmpty(from, to, std::move(done)); - }); -} - -void Database::moveIfEmpty( - const Key &from, - const Key &to, - FnMut &&done) { - _wrapped.with([ - from, - to, - done = std::move(done) - ](Implementation &unwrapped) mutable { - unwrapped.moveIfEmpty(from, to, std::move(done)); - }); -} - -void Database::put( - const Key &key, - TaggedValue &&value, - FnMut &&done) { - _wrapped.with([ - key, - value = std::move(value), - done = std::move(done) - ](Implementation &unwrapped) mutable { - unwrapped.put(key, std::move(value), std::move(done)); - }); -} - -void Database::putIfEmpty( - const Key &key, - TaggedValue &&value, - FnMut &&done) { - _wrapped.with([ - key, - value = std::move(value), - done = std::move(done) - ](Implementation &unwrapped) mutable { - unwrapped.putIfEmpty(key, std::move(value), std::move(done)); - }); -} - -void Database::getWithTag( - const Key &key, - FnMut &&done) { - _wrapped.with([ - key, - done = std::move(done) - ](Implementation &unwrapped) mutable { - unwrapped.get(key, std::move(done)); - }); -} - -void Database::getWithSizes( - const Key &key, - std::vector &&keys, - FnMut&&)> &&done) { - _wrapped.with([ - key, - keys = std::move(keys), - done = std::move(done) - ](Implementation &unwrapped) mutable { - unwrapped.getWithSizes(key, std::move(keys), std::move(done)); - }); -} - -auto Database::statsOnMain() const -> rpl::producer { - return _wrapped.producer_on_main([](const Implementation &unwrapped) { - return unwrapped.stats(); - }); -} - -void Database::clear(FnMut &&done) { - _wrapped.with([ - done = std::move(done) - ](Implementation &unwrapped) mutable { - unwrapped.clear(std::move(done)); - }); -} - -void Database::clearByTag(uint8 tag, FnMut &&done) { - _wrapped.with([ - tag, - done = std::move(done) - ](Implementation &unwrapped) mutable { - unwrapped.clearByTag(tag, std::move(done)); - }); -} - -void Database::sync() { - auto semaphore = crl::semaphore(); - _wrapped.with([&](Implementation &) { - semaphore.release(); - }); - semaphore.acquire(); -} - -Database::~Database() = default; - -} // namespace Cache -} // namespace Storage diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_database.h b/Telegram/SourceFiles/storage/cache/storage_cache_database.h deleted file mode 100644 index c7998fd2a..000000000 --- a/Telegram/SourceFiles/storage/cache/storage_cache_database.h +++ /dev/null @@ -1,91 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#pragma once - -#include "storage/cache/storage_cache_types.h" -#include "base/basic_types.h" -#include -#include -#include -#include - -namespace Storage { -class EncryptionKey; -namespace Cache { -namespace details { -class DatabaseObject; -} // namespace details - -class Database { -public: - using Settings = details::Settings; - using SettingsUpdate = details::SettingsUpdate; - Database(const QString &path, const Settings &settings); - - void reconfigure(const Settings &settings); - void updateSettings(const SettingsUpdate &update); - - void open(EncryptionKey &&key, FnMut &&done = nullptr); - void close(FnMut &&done = nullptr); - - void put( - const Key &key, - QByteArray &&value, - FnMut &&done = nullptr); - void get(const Key &key, FnMut &&done); - void remove(const Key &key, FnMut &&done = nullptr); - - void putIfEmpty( - const Key &key, - QByteArray &&value, - FnMut &&done = nullptr); - void copyIfEmpty( - const Key &from, - const Key &to, - FnMut &&done = nullptr); - void moveIfEmpty( - const Key &from, - const Key &to, - FnMut &&done = nullptr); - - using TaggedValue = details::TaggedValue; - void put( - const Key &key, - TaggedValue &&value, - FnMut &&done = nullptr); - void putIfEmpty( - const Key &key, - TaggedValue &&value, - FnMut &&done = nullptr); - void getWithTag(const Key &key, FnMut &&done); - - void getWithSizes( - const Key &key, - std::vector &&keys, - FnMut&&)> &&done); - - using Stats = details::Stats; - using TaggedSummary = details::TaggedSummary; - rpl::producer statsOnMain() const; - - void clear(FnMut &&done = nullptr); - void clearByTag(uint8 tag, FnMut &&done = nullptr); - void waitForCleaner(FnMut &&done = nullptr); - - void sync(); - - ~Database(); - -private: - using Implementation = details::DatabaseObject; - crl::object_on_queue _wrapped; - -}; - -} // namespace Cache -} // namespace Storage diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_database_object.cpp b/Telegram/SourceFiles/storage/cache/storage_cache_database_object.cpp deleted file mode 100644 index ccdeda4ce..000000000 --- a/Telegram/SourceFiles/storage/cache/storage_cache_database_object.cpp +++ /dev/null @@ -1,1328 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#include "storage/cache/storage_cache_database_object.h" - -#include "storage/cache/storage_cache_cleaner.h" -#include "storage/cache/storage_cache_compactor.h" -#include "storage/cache/storage_cache_binlog_reader.h" -#include "storage/storage_encryption.h" -#include "storage/storage_encrypted_file.h" -#include "base/flat_map.h" -#include "base/algorithm.h" -#include -#include -#include -#include -#include - -namespace Storage { -namespace Cache { -namespace details { -namespace { - -constexpr auto kMaxDelayAfterFailure = 24 * 60 * 60 * crl::time(1000); - -uint32 CountChecksum(bytes::const_span data) { - const auto seed = uint32(0); - return XXH32(data.data(), data.size(), seed); -} - -QString PlaceFromId(PlaceId place) { - auto result = QString(); - result.reserve(15); - const auto pushDigit = [&](uint8 digit) { - const auto hex = (digit < 0x0A) - ? char('0' + digit) - : char('A' + (digit - 0x0A)); - result.push_back(hex); - }; - const auto push = [&](uint8 value) { - pushDigit(value & 0x0F); - pushDigit(value >> 4); - }; - for (auto i = 0; i != place.size(); ++i) { - push(place[i]); - if (!i) { - result.push_back('/'); - } - } - return result; -} - -int32 GetUnixtime() { - return std::max(int32(time(nullptr)), 1); -} - -} // namespace - -DatabaseObject::Entry::Entry( - PlaceId place, - uint8 tag, - uint32 checksum, - size_type size, - uint64 useTime) -: useTime(useTime) -, size(size) -, checksum(checksum) -, place(place) -, tag(tag) { -} - -DatabaseObject::DatabaseObject( - crl::weak_on_queue weak, - const QString &path, - const Settings &settings) -: _weak(std::move(weak)) -, _base(ComputeBasePath(path)) -, _settings(settings) -, _writeBundlesTimer(_weak, [=] { writeBundles(); checkCompactor(); }) -, _pruneTimer(_weak, [=] { prune(); }) { - checkSettings(); -} - -void DatabaseObject::reconfigure(const Settings &settings) { - Expects(_key.empty()); - - _settings = settings; - checkSettings(); -} - -void DatabaseObject::updateSettings(const SettingsUpdate &update) { - _settings.totalSizeLimit = update.totalSizeLimit; - _settings.totalTimeLimit = update.totalTimeLimit; - checkSettings(); - - optimize(); -} - -void DatabaseObject::checkSettings() { - Expects(_settings.staleRemoveChunk > 0); - Expects(_settings.maxDataSize > 0 - && _settings.maxDataSize < kDataSizeLimit); - Expects(_settings.maxBundledRecords > 0 - && _settings.maxBundledRecords < kBundledRecordsLimit); - Expects(!_settings.totalTimeLimit - || _settings.totalTimeLimit > 0); - Expects(!_settings.totalSizeLimit - || _settings.totalSizeLimit > _settings.maxDataSize); -} - -template -void DatabaseObject::invokeCallback( - Callback &&callback, - Args &&...args) const { - if (callback) { - callback(std::move(args)...); - } -} - -Error DatabaseObject::ioError(const QString &path) const { - return { Error::Type::IO, path }; -} - -void DatabaseObject::open(EncryptionKey &&key, FnMut &&done) { - close(nullptr); - - const auto error = openSomeBinlog(std::move(key)); - if (error.type != Error::Type::None) { - close(nullptr); - } - invokeCallback(done, error); -} - -Error DatabaseObject::openSomeBinlog(EncryptionKey &&key) { - const auto version = readVersion(); - const auto result = openBinlog(version, File::Mode::ReadAppend, key); - switch (result) { - case File::Result::Success: return Error::NoError(); - case File::Result::Failed: return openNewBinlog(key); - case File::Result::LockFailed: - return Error{ Error::Type::LockFailed, binlogPath(version) }; - case File::Result::WrongKey: - return _settings.clearOnWrongKey - ? openNewBinlog(key) - : Error{ Error::Type::WrongKey, binlogPath(version) }; - } - Unexpected("Result from DatabaseObject::openBinlog."); -} - -Error DatabaseObject::openNewBinlog(EncryptionKey &key) { - const auto available = findAvailableVersion(); - if (!writeVersion(available)) { - return ioError(versionPath()); - } - const auto open = openBinlog(available, File::Mode::Write, key); - if (open != File::Result::Success) { - return ioError(binlogPath(available)); - } - return Error::NoError(); -} - -QString DatabaseObject::computePath(Version version) const { - return _base + QString::number(version) + '/'; -} - -QString DatabaseObject::BinlogFilename() { - return QStringLiteral("binlog"); -} - -QString DatabaseObject::CompactReadyFilename() { - return QStringLiteral("binlog-ready"); -} - -QString DatabaseObject::binlogPath(Version version) const { - return computePath(version) + BinlogFilename(); -} - -QString DatabaseObject::binlogPath() const { - return _path + BinlogFilename(); -} - -QString DatabaseObject::compactReadyPath(Version version) const { - return computePath(version) + CompactReadyFilename(); -} - -QString DatabaseObject::compactReadyPath() const { - return _path + CompactReadyFilename(); -} - -File::Result DatabaseObject::openBinlog( - Version version, - File::Mode mode, - EncryptionKey &key) { - const auto ready = compactReadyPath(version); - const auto path = binlogPath(version); - if (QFile(ready).exists() && !File::Move(ready, path)) { - return File::Result::Failed; - } - const auto result = _binlog.open(path, mode, key); - if (result != File::Result::Success) { - return result; - } - const auto headerRequired = (mode == File::Mode::Read) - || (mode == File::Mode::ReadAppend && _binlog.size() > 0); - const auto headerResult = headerRequired ? readHeader() : writeHeader(); - if (!headerResult) { - return File::Result::Failed; - } - _path = computePath(version); - _key = std::move(key); - createCleaner(); - readBinlog(); - return File::Result::Success; -} - -bool DatabaseObject::readHeader() { - if (const auto header = BinlogWrapper::ReadHeader(_binlog, _settings)) { - _time.setRelative((_time.system = header->systemTime)); - return true; - } - return false; -} - -bool DatabaseObject::writeHeader() { - auto header = BasicHeader(); - const auto now = _settings.trackEstimatedTime ? GetUnixtime() : 0; - _time.setRelative((_time.system = header.systemTime = now)); - if (_settings.trackEstimatedTime) { - header.flags |= header.kTrackEstimatedTime; - } - return _binlog.write(bytes::object_as_span(&header)); -} - -template -void DatabaseObject::readBinlogHelper( - Reader &reader, - Handlers &&...handlers) { - while (true) { - const auto done = reader.readTillEnd( - std::forward(handlers)...); - if (done) { - break; - } - } -} - -void DatabaseObject::readBinlog() { - BinlogWrapper wrapper(_binlog, _settings); - if (_settings.trackEstimatedTime) { - BinlogReader< - StoreWithTime, - MultiStoreWithTime, - MultiRemove, - MultiAccess> reader(wrapper); - readBinlogHelper(reader, [&](const StoreWithTime &record) { - return processRecordStore( - &record, - std::is_class{}); - }, [&](const MultiStoreWithTime &header, const auto &element) { - return processRecordMultiStore(header, element); - }, [&](const MultiRemove &header, const auto &element) { - return processRecordMultiRemove(header, element); - }, [&](const MultiAccess &header, const auto &element) { - return processRecordMultiAccess(header, element); - }); - } else { - BinlogReader< - Store, - MultiStore, - MultiRemove> reader(wrapper); - readBinlogHelper(reader, [&](const Store &record) { - return processRecordStore(&record, std::is_class{}); - }, [&](const MultiStore &header, const auto &element) { - return processRecordMultiStore(header, element); - }, [&](const MultiRemove &header, const auto &element) { - return processRecordMultiRemove(header, element); - }); - } - adjustRelativeTime(); - optimize(); -} - -uint64 DatabaseObject::countRelativeTime() const { - const auto now = GetUnixtime(); - const auto delta = std::max(int64(now) - int64(_time.system), 0LL); - return _time.getRelative() + delta; -} - -uint64 DatabaseObject::pruneBeforeTime() const { - const auto relative = countRelativeTime(); - return (_settings.totalTimeLimit && relative > _settings.totalTimeLimit) - ? (relative - _settings.totalTimeLimit) - : 0ULL; -} - -void DatabaseObject::optimize() { - if (!startDelayedPruning()) { - checkCompactor(); - } -} - -bool DatabaseObject::startDelayedPruning() { - if (!_settings.trackEstimatedTime || _map.empty()) { - return false; - } - const auto before = pruneBeforeTime(); - const auto pruning = [&] { - if (_settings.totalSizeLimit > 0 - && _totalSize > _settings.totalSizeLimit) { - return true; - } else if ((!_minimalEntryTime && !_map.empty()) - || _minimalEntryTime <= before) { - return true; - } - return false; - }(); - if (pruning) { - if (!_pruneTimer.isActive() - || _pruneTimer.remainingTime() > _settings.pruneTimeout) { - _pruneTimer.callOnce(_settings.pruneTimeout); - } - return true; - } else if (_minimalEntryTime != 0) { - Assert(_minimalEntryTime > before); - const auto seconds = int64(_minimalEntryTime - before); - if (!_pruneTimer.isActive()) { - _pruneTimer.callOnce(std::min( - crl::time(seconds * 1000), - _settings.maxPruneCheckTimeout)); - } - } - return false; -} - -void DatabaseObject::prune() { - if (!_stale.empty()) { - return; - } - auto stale = base::flat_set(); - auto staleTotalSize = int64(); - collectTimeStale(stale, staleTotalSize); - collectSizeStale(stale, staleTotalSize); - if (stale.size() <= _settings.staleRemoveChunk) { - clearStaleNow(stale); - } else { - _stale = ranges::view::all(stale) | ranges::to_vector; - startStaleClear(); - } -} - -void DatabaseObject::startStaleClear() { - // Report "Clearing..." status. - pushStats(); - clearStaleChunk(); -} - -void DatabaseObject::clearStaleNow(const base::flat_set &stale) { - if (stale.empty()) { - return; - } - - // Report "Clearing..." status. - _stale.push_back(stale.front()); - pushStats(); - - for (const auto &key : stale) { - remove(key, nullptr); - } - - // Report correct status async. - _stale.clear(); - optimize(); -} - -void DatabaseObject::clearStaleChunkDelayed() { - if (_clearingStale) { - return; - } - _clearingStale = true; - _weak.with([](DatabaseObject &that) { - if (base::take(that._clearingStale)) { - that.clearStaleChunk(); - } - }); -} - -void DatabaseObject::clearStaleChunk() { - if (_stale.empty()) { - return; - } - const auto stale = gsl::make_span(_stale); - const auto count = size_type(stale.size()); - const auto clear = std::min(count, _settings.staleRemoveChunk); - for (const auto &key : stale.subspan(count - clear)) { - remove(key, nullptr); - } - _stale.resize(count - clear); - if (_stale.empty()) { - base::take(_stale); - optimize(); - } else { - clearStaleChunkDelayed(); - } -} - -void DatabaseObject::collectTimeStale( - base::flat_set &stale, - int64 &staleTotalSize) { - if (!_settings.totalTimeLimit) { - return; - } - const auto before = pruneBeforeTime(); - if (!_minimalEntryTime || _minimalEntryTime > before) { - return; - } - _minimalEntryTime = 0; - _entriesWithMinimalTimeCount = 0; - for (const auto &[key, entry] : _map) { - if (entry.useTime <= before) { - stale.emplace(key); - staleTotalSize += entry.size; - } else if (!_minimalEntryTime - || _minimalEntryTime > entry.useTime) { - _minimalEntryTime = entry.useTime; - _entriesWithMinimalTimeCount = 1; - } else if (_minimalEntryTime == entry.useTime) { - ++_entriesWithMinimalTimeCount; - } - } -} - -void DatabaseObject::collectSizeStale( - base::flat_set &stale, - int64 &staleTotalSize) { - const auto removeSize = (_settings.totalSizeLimit > 0) - ? (_totalSize - staleTotalSize - _settings.totalSizeLimit) - : 0; - if (removeSize <= 0) { - return; - } - - using Bucket = std::pair; - auto oldest = base::flat_multi_map< - int64, - const Bucket*, - std::greater<>>(); - auto oldestTotalSize = int64(); - - const auto canRemoveFirst = [&](const Entry &adding) { - const auto totalSizeAfterAdd = oldestTotalSize + adding.size; - const auto &first = oldest.begin()->second->second; - return (adding.useTime <= first.useTime - && (totalSizeAfterAdd - removeSize >= first.size)); - }; - - for (const auto &bucket : _map) { - const auto &entry = bucket.second; - if (stale.contains(bucket.first)) { - continue; - } - const auto add = (oldestTotalSize < removeSize) - ? true - : (entry.useTime < oldest.begin()->second->second.useTime); - if (!add) { - continue; - } - while (!oldest.empty() && canRemoveFirst(entry)) { - oldestTotalSize -= oldest.begin()->second->second.size; - oldest.erase(oldest.begin()); - } - oldestTotalSize += entry.size; - oldest.emplace(entry.useTime, &bucket); - } - - for (const auto &pair : oldest) { - stale.emplace(pair.second->first); - } - staleTotalSize += oldestTotalSize; -} - -void DatabaseObject::adjustRelativeTime() { - if (!_settings.trackEstimatedTime) { - return; - } - const auto now = GetUnixtime(); - if (now < _time.system) { - writeMultiAccessBlock(); - } -} - -template -bool DatabaseObject::processRecordStoreGeneric( - const Record *record, - Postprocess &&postprocess) { - const auto size = record->getSize(); - if (size <= 0 || size > _settings.maxDataSize) { - return false; - } - auto entry = Entry( - record->place, - record->tag, - record->checksum, - size, - _time.getRelative()); - if (!postprocess(entry, record)) { - return false; - } - setMapEntry(record->key, std::move(entry)); - return true; -} - -bool DatabaseObject::processRecordStore( - const Store *record, - std::is_class) { - const auto postprocess = [](auto&&...) { return true; }; - return processRecordStoreGeneric(record, postprocess); -} - -bool DatabaseObject::processRecordStore( - const StoreWithTime *record, - std::is_class) { - const auto postprocess = [&]( - Entry &entry, - not_null record) { - applyTimePoint(record->time); - entry.useTime = record->time.getRelative(); - return true; - }; - return processRecordStoreGeneric(record, postprocess); -} - -template -bool DatabaseObject::processRecordMultiStore( - const Record &header, - const GetElement &element) { - while (const auto entry = element()) { - if (!processRecordStore( - entry, - std::is_class{})) { - return false; - } - } - return true; -} - -template -bool DatabaseObject::processRecordMultiRemove( - const MultiRemove &header, - const GetElement &element) { - _binlogExcessLength += sizeof(header); - while (const auto entry = element()) { - _binlogExcessLength += sizeof(*entry); - if (const auto i = _map.find(*entry); i != end(_map)) { - eraseMapEntry(i); - } - } - return true; -} - -template -bool DatabaseObject::processRecordMultiAccess( - const MultiAccess &header, - const GetElement &element) { - Expects(_settings.trackEstimatedTime); - - applyTimePoint(header.time); - const auto relative = header.time.getRelative(); - - _binlogExcessLength += sizeof(header); - while (const auto entry = element()) { - _binlogExcessLength += sizeof(*entry); - if (const auto i = _map.find(*entry); i != end(_map)) { - i->second.useTime = relative; - } - } - return true; -} - -void DatabaseObject::setMapEntry(const Key &key, Entry &&entry) { - auto &already = _map[key]; - updateStats(already, entry); - if (already.size != 0) { - _binlogExcessLength += _settings.trackEstimatedTime - ? sizeof(StoreWithTime) - : sizeof(Store); - } - if (entry.useTime != 0 - && (entry.useTime < _minimalEntryTime || !_minimalEntryTime)) { - _minimalEntryTime = entry.useTime; - _entriesWithMinimalTimeCount = 1; - } else if (_minimalEntryTime != 0 && already.useTime != entry.useTime) { - if (entry.useTime == _minimalEntryTime) { - Assert(_entriesWithMinimalTimeCount > 0); - ++_entriesWithMinimalTimeCount; - } else if (already.useTime == _minimalEntryTime) { - Assert(_entriesWithMinimalTimeCount > 0); - if (!--_entriesWithMinimalTimeCount) { - _minimalEntryTime = 0; - } - } - } - already = std::move(entry); -} - -void DatabaseObject::updateStats(const Entry &was, const Entry &now) { - _totalSize += now.size - was.size; - if (now.tag == was.tag) { - if (now.tag) { - auto &summary = _taggedStats[now.tag]; - summary.count += (now.size ? 1 : 0) - (was.size ? 1 : 0); - summary.totalSize += now.size - was.size; - } - } else { - if (now.tag) { - auto &summary = _taggedStats[now.tag]; - summary.count += (now.size ? 1 : 0); - summary.totalSize += now.size; - } - if (was.tag) { - auto &summary = _taggedStats[was.tag]; - summary.count -= (was.size ? 1 : 0); - summary.totalSize -= was.size; - } - } - pushStatsDelayed(); -} - -void DatabaseObject::pushStatsDelayed() { - if (_pushingStats) { - return; - } - _pushingStats = true; - _weak.with([](DatabaseObject &that) { - if (base::take(that._pushingStats)) { - that.pushStats(); - } - }); -} - -void DatabaseObject::pushStats() { - if (_stats.has_consumers()) { - _stats.fire(collectStats()); - } -} - -void DatabaseObject::eraseMapEntry(const Map::const_iterator &i) { - if (i != end(_map)) { - const auto &entry = i->second; - updateStats(entry, Entry()); - if (_minimalEntryTime != 0 && entry.useTime == _minimalEntryTime) { - Assert(_entriesWithMinimalTimeCount > 0); - if (!--_entriesWithMinimalTimeCount) { - _minimalEntryTime = 0; - } - } - _map.erase(i); - } -} - -EstimatedTimePoint DatabaseObject::countTimePoint() const { - const auto now = GetUnixtime(); - const auto delta = std::max(int64(now) - int64(_time.system), 0LL); - auto result = EstimatedTimePoint(); - result.system = now; - result.setRelative(_time.getRelative() + delta); - return result; -} - -void DatabaseObject::applyTimePoint(EstimatedTimePoint time) { - const auto possible = time.getRelative(); - const auto current = _time.getRelative(); - if (possible > current) { - _time = time; - } -} - -void DatabaseObject::compactorDone( - const QString &path, - int64 originalReadTill) { - const auto size = _binlog.size(); - const auto binlog = binlogPath(); - const auto ready = compactReadyPath(); - if (originalReadTill != size) { - originalReadTill = CatchUp( - path, - binlog, - _key, - originalReadTill, - _settings.readBlockSize); - if (originalReadTill != size) { - compactorFail(); - return; - } - } - if (!File::Move(path, ready)) { - compactorFail(); - return; - } - const auto guard = gsl::finally([&] { - _compactor = CompactorWrap(); - }); - _binlog.close(); - if (!File::Move(ready, binlog)) { - compactorFail(); - return; - } - const auto result = _binlog.open(binlog, File::Mode::ReadAppend, _key); - if (result != File::Result::Success) { - compactorFail(); - return; - } else if (!_binlog.seek(_binlog.size())) { - _binlog.close(); - compactorFail(); - return; - } - _binlogExcessLength -= _compactor.excessLength; - Assert(_binlogExcessLength >= 0); -} - -void DatabaseObject::compactorFail() { - const auto delay = _compactor.delayAfterFailure; - _compactor = CompactorWrap(); - _compactor.nextAttempt = crl::now() + delay; - _compactor.delayAfterFailure = std::min( - delay * 2, - kMaxDelayAfterFailure); - QFile(compactReadyPath()).remove(); -} - -void DatabaseObject::close(FnMut &&done) { - if (_binlog.isOpen()) { - writeBundles(); - _binlog.close(); - } - invokeCallback(done); - clearState(); -} - -void DatabaseObject::clearState() { - _path = QString(); - _key = {}; - _map = {}; - _removing = {}; - _accessed = {}; - _stale = {}; - _time = {}; - _binlogExcessLength = 0; - _totalSize = 0; - _minimalEntryTime = 0; - _entriesWithMinimalTimeCount = 0; - _taggedStats = {}; - _pushingStats = false; - _writeBundlesTimer.cancel(); - _pruneTimer.cancel(); - _compactor = CompactorWrap(); -} - -void DatabaseObject::put( - const Key &key, - TaggedValue &&value, - FnMut &&done) { - if (value.bytes.isEmpty()) { - remove(key, std::move(done)); - return; - } - _removing.erase(key); - _stale.erase(ranges::remove(_stale, key), end(_stale)); - - const auto checksum = CountChecksum(bytes::make_span(value.bytes)); - const auto maybepath = writeKeyPlace(key, value, checksum); - if (!maybepath) { - invokeCallback(done, ioError(binlogPath())); - return; - } else if (maybepath->isEmpty()) { - // Nothing changed. - invokeCallback(done, Error::NoError()); - recordEntryAccess(key); - return; - } - const auto path = *maybepath; - File data; - const auto result = data.open(path, File::Mode::Write, _key); - switch (result) { - case File::Result::Failed: - remove(key, nullptr); - invokeCallback(done, ioError(path)); - break; - - case File::Result::LockFailed: - remove(key, nullptr); - invokeCallback(done, Error{ Error::Type::LockFailed, path }); - break; - - case File::Result::Success: { - const auto success = data.writeWithPadding( - bytes::make_detached_span(value.bytes)); - if (!success) { - data.close(); - remove(key, nullptr); - invokeCallback(done, ioError(path)); - } else { - data.flush(); - invokeCallback(done, Error::NoError()); - optimize(); - } - } break; - - default: Unexpected("Result in DatabaseObject::put."); - } -} - -template -std::optional DatabaseObject::writeKeyPlaceGeneric( - StoreRecord &&record, - const Key &key, - const TaggedValue &value, - uint32 checksum) { - Expects(value.bytes.size() <= _settings.maxDataSize); - - const auto size = size_type(value.bytes.size()); - record.tag = value.tag; - record.key = key; - record.setSize(size); - record.checksum = checksum; - if (const auto i = _map.find(key); i != end(_map)) { - const auto &already = i->second; - if (already.tag == record.tag - && already.size == size - && already.checksum == checksum - && readValueData(already.place, size) == value.bytes) { - return QString(); - } - record.place = already.place; - } else { - do { - bytes::set_random(bytes::object_as_span(&record.place)); - } while (!isFreePlace(record.place)); - } - const auto result = placePath(record.place); - auto writeable = record; - const auto success = _binlog.write(bytes::object_as_span(&writeable)); - if (!success) { - _binlog.close(); - return QString(); - } - _binlog.flush(); - - const auto applied = processRecordStore( - &record, - std::is_class{}); - Assert(applied); - return result; -} - -std::optional DatabaseObject::writeKeyPlace( - const Key &key, - const TaggedValue &data, - uint32 checksum) { - if (!_settings.trackEstimatedTime) { - return writeKeyPlaceGeneric(Store(), key, data, checksum); - } - auto record = StoreWithTime(); - record.time = countTimePoint(); - const auto writing = record.time.getRelative(); - const auto current = _time.getRelative(); - Assert(writing >= current); - if ((writing - current) * crl::time(1000) - < _settings.writeBundleDelay) { - // We don't want to produce a lot of unique _time.relative values. - // So if change in it is not large we stick to the old value. - record.time = _time; - } - return writeKeyPlaceGeneric(std::move(record), key, data, checksum); -} - -template -Error DatabaseObject::writeExistingPlaceGeneric( - StoreRecord &&record, - const Key &key, - const Entry &entry) { - record.key = key; - record.tag = entry.tag; - record.setSize(entry.size); - record.checksum = entry.checksum; - if (const auto i = _map.find(key); i != end(_map)) { - const auto &already = i->second; - if (already.tag == record.tag - && already.size == entry.size - && already.checksum == entry.checksum - && (readValueData(already.place, already.size) - == readValueData(entry.place, entry.size))) { - return Error::NoError(); - } - } - record.place = entry.place; - auto writeable = record; - const auto success = _binlog.write(bytes::object_as_span(&writeable)); - if (!success) { - _binlog.close(); - return ioError(binlogPath()); - } - _binlog.flush(); - - const auto applied = processRecordStore( - &record, - std::is_class{}); - Assert(applied); - return Error::NoError(); -} - -Error DatabaseObject::writeExistingPlace( - const Key &key, - const Entry &entry) { - if (!_settings.trackEstimatedTime) { - return writeExistingPlaceGeneric(Store(), key, entry); - } - auto record = StoreWithTime(); - record.time = countTimePoint(); - const auto writing = record.time.getRelative(); - const auto current = _time.getRelative(); - Assert(writing >= current); - if ((writing - current) * crl::time(1000) - < _settings.writeBundleDelay) { - // We don't want to produce a lot of unique _time.relative values. - // So if change in it is not large we stick to the old value. - record.time = _time; - } - return writeExistingPlaceGeneric(std::move(record), key, entry); -} - -void DatabaseObject::get( - const Key &key, - FnMut &&done) { - const auto i = _map.find(key); - if (i == _map.end()) { - invokeCallback(done, TaggedValue()); - return; - } - const auto &entry = i->second; - - auto bytes = readValueData(entry.place, entry.size); - if (bytes.isEmpty()) { - remove(key, nullptr); - invokeCallback(done, TaggedValue()); - } else if (CountChecksum(bytes::make_span(bytes)) != entry.checksum) { - remove(key, nullptr); - invokeCallback(done, TaggedValue()); - } else { - invokeCallback(done, TaggedValue(std::move(bytes), entry.tag)); - recordEntryAccess(key); - } -} - -void DatabaseObject::getWithSizes( - const Key &key, - std::vector &&keys, - FnMut&&)> &&done) { - get(key, [&](TaggedValue &&value) { - if (value.bytes.isEmpty()) { - invokeCallback(done, QByteArray(), std::vector()); - return; - } - - auto sizes = keys | ranges::view::transform([&](const Key &sizeKey) { - const auto i = _map.find(sizeKey); - return (i != end(_map)) ? int(i->second.size) : 0; - }) | ranges::to_vector; - - invokeCallback(done, std::move(value.bytes), std::move(sizes)); - }); -} - -QByteArray DatabaseObject::readValueData( - PlaceId place, - size_type size) const { - const auto path = placePath(place); - File data; - const auto result = data.open(path, File::Mode::Read, _key); - switch (result) { - case File::Result::Failed: - case File::Result::WrongKey: return QByteArray(); - case File::Result::Success: { - auto result = QByteArray(size, Qt::Uninitialized); - const auto bytes = bytes::make_detached_span(result); - const auto read = data.readWithPadding(bytes); - if (read != size) { - return QByteArray(); - } - return result; - } break; - } - Unexpected("Result in DatabaseObject::get."); -} - -void DatabaseObject::recordEntryAccess(const Key &key) { - if (!_settings.trackEstimatedTime) { - return; - } - _accessed.emplace(key); - writeMultiAccessLazy(); - optimize(); -} - -void DatabaseObject::remove(const Key &key, FnMut &&done) { - const auto i = _map.find(key); - if (i != _map.end()) { - _removing.emplace(key); - writeMultiRemoveLazy(); - - const auto path = placePath(i->second.place); - eraseMapEntry(i); - if (QFile(path).remove() || !QFile(path).exists()) { - invokeCallback(done, Error::NoError()); - } else { - invokeCallback(done, ioError(path)); - } - } else { - invokeCallback(done, Error::NoError()); - } -} - -void DatabaseObject::putIfEmpty( - const Key &key, - TaggedValue &&value, - FnMut &&done) { - if (_map.find(key) != end(_map)) { - invokeCallback(done, Error::NoError()); - return; - } - put(key, std::move(value), std::move(done)); -} - -void DatabaseObject::copyIfEmpty( - const Key &from, - const Key &to, - FnMut &&done) { - if (_map.find(to) != end(_map)) { - invokeCallback(done, Error::NoError()); - return; - } - get(from, [&](TaggedValue &&value) { - put(to, std::move(value), std::move(done)); - }); -} - -void DatabaseObject::moveIfEmpty( - const Key &from, - const Key &to, - FnMut &&done) { - if (_map.find(to) != end(_map)) { - invokeCallback(done, Error::NoError()); - return; - } - const auto i = _map.find(from); - if (i == _map.end()) { - invokeCallback(done, Error::NoError()); - return; - } - _removing.emplace(from); - - const auto entry = i->second; - eraseMapEntry(i); - - const auto result = writeMultiRemove(); - if (result.type != Error::Type::None) { - invokeCallback(done, result); - return; - } - _removing.erase(to); - _stale.erase(ranges::remove(_stale, to), end(_stale)); - invokeCallback(done, writeExistingPlace(to, entry)); -} - -rpl::producer DatabaseObject::stats() const { - return _stats.events_starting_with(collectStats()); -} - -Stats DatabaseObject::collectStats() const { - auto result = Stats(); - result.tagged = _taggedStats; - result.full.count = _map.size(); - result.full.totalSize = _totalSize; - result.clearing = (_cleaner.object != nullptr) || !_stale.empty(); - return result; -} - -void DatabaseObject::writeBundlesLazy() { - if (!_writeBundlesTimer.isActive()) { - _writeBundlesTimer.callOnce(_settings.writeBundleDelay); - } -} - -void DatabaseObject::writeMultiRemoveLazy() { - if (_removing.size() == _settings.maxBundledRecords) { - writeMultiRemove(); - } else { - writeBundlesLazy(); - } -} - -Error DatabaseObject::writeMultiRemove() { - Expects(_removing.size() <= _settings.maxBundledRecords); - - if (_removing.empty()) { - return Error::NoError(); - } - const auto size = _removing.size(); - auto header = MultiRemove(size); - auto list = std::vector(); - list.reserve(size); - for (const auto &key : base::take(_removing)) { - list.push_back(key); - } - if (_binlog.write(bytes::object_as_span(&header)) - && _binlog.write(bytes::make_span(list))) { - _binlog.flush(); - _binlogExcessLength += bytes::object_as_span(&header).size() - + bytes::make_span(list).size(); - return Error::NoError(); - } - _binlog.close(); - return ioError(binlogPath()); -} - -void DatabaseObject::writeMultiAccessLazy() { - if (_accessed.size() == _settings.maxBundledRecords) { - writeMultiAccess(); - } else { - writeBundlesLazy(); - } -} - -Error DatabaseObject::writeMultiAccess() { - if (_accessed.empty()) { - return Error::NoError(); - } - return writeMultiAccessBlock(); -} - -Error DatabaseObject::writeMultiAccessBlock() { - Expects(_settings.trackEstimatedTime); - Expects(_accessed.size() <= _settings.maxBundledRecords); - - const auto time = countTimePoint(); - const auto size = _accessed.size(); - auto header = MultiAccess(time, size); - auto list = std::vector(); - if (size > 0) { - list.reserve(size); - for (const auto &key : base::take(_accessed)) { - list.push_back(key); - } - } - _time = time; - for (const auto &entry : list) { - if (const auto i = _map.find(entry); i != end(_map)) { - i->second.useTime = _time.getRelative(); - } - } - - if (_binlog.write(bytes::object_as_span(&header)) - && (!size || _binlog.write(bytes::make_span(list)))) { - _binlog.flush(); - _binlogExcessLength += bytes::object_as_span(&header).size() - + bytes::make_span(list).size(); - return Error::NoError(); - } - _binlog.close(); - return ioError(binlogPath()); -} - -void DatabaseObject::writeBundles() { - writeMultiRemove(); - if (_settings.trackEstimatedTime) { - writeMultiAccess(); - } -} - -void DatabaseObject::createCleaner() { - auto done = [weak = _weak](Error error) { - weak.with([=](DatabaseObject &that) { - that.cleanerDone(error); - }); - }; - _cleaner.object = std::make_unique( - _base, - _cleaner.guard.make_guard(), - std::move(done)); - pushStatsDelayed(); -} - -void DatabaseObject::cleanerDone(Error error) { - invokeCallback(_cleaner.done); - _cleaner = CleanerWrap(); - pushStatsDelayed(); -} - -void DatabaseObject::checkCompactor() { - if (_compactor.object - || !_settings.compactAfterExcess - || _binlogExcessLength < _settings.compactAfterExcess) { - return; - } else if (_settings.compactAfterFullSize - && (_binlogExcessLength * _settings.compactAfterFullSize - < _settings.compactAfterExcess * _binlog.size())) { - return; - } else if (crl::now() < _compactor.nextAttempt || !_binlog.isOpen()) { - return; - } - auto info = Compactor::Info(); - info.till = _binlog.size(); - info.systemTime = _time.system; - info.keysCount = _map.size(); - _compactor.object = std::make_unique( - _weak, - _compactor.guard.make_guard(), - _path, - _settings, - base::duplicate(_key), - info); - _compactor.excessLength = _binlogExcessLength; -} - -void DatabaseObject::clear(FnMut &&done) { - auto key = std::move(_key); - if (!key.empty()) { - close(nullptr); - } - const auto version = findAvailableVersion(); - if (!writeVersion(version)) { - invokeCallback(done, ioError(versionPath())); - return; - } - if (key.empty()) { - invokeCallback(done, Error::NoError()); - createCleaner(); - return; - } - open(std::move(key), std::move(done)); -} - -void DatabaseObject::clearByTag(uint8 tag, FnMut &&done) { - const auto hadStale = !_stale.empty(); - for (const auto &[key, entry] : _map) { - if (entry.tag == tag) { - _stale.push_back(key); - } - } - if (!hadStale) { - startStaleClear(); - } - invokeCallback(done, Error::NoError()); -} - -void DatabaseObject::waitForCleaner(FnMut &&done) { - while (!_stale.empty()) { - clearStaleChunk(); - } - if (_cleaner.object) { - _cleaner.done = std::move(done); - } else { - invokeCallback(done); - } -} - -auto DatabaseObject::getManyRaw(const std::vector &keys) const --> std::vector { - auto result = std::vector(); - result.reserve(keys.size()); - for (const auto &key : keys) { - if (const auto i = _map.find(key); i != end(_map)) { - result.push_back(*i); - } - } - return result; -} - -DatabaseObject::~DatabaseObject() { - close(nullptr); -} - -auto DatabaseObject::findAvailableVersion() const -> Version { - const auto entries = QDir(_base).entryList( - QDir::Dirs | QDir::NoDotAndDotDot); - auto versions = base::flat_set(); - for (const auto entry : entries) { - versions.insert(entry.toInt()); - } - auto result = Version(); - for (const auto version : versions) { - if (result != version) { - break; - } - ++result; - } - return result; -} - -QString DatabaseObject::versionPath() const { - return VersionFilePath(_base); -} - -bool DatabaseObject::writeVersion(Version version) { - return WriteVersionValue(_base, version); -} - -auto DatabaseObject::readVersion() const -> Version { - if (const auto result = ReadVersionValue(_base)) { - return *result; - } - return Version(); -} - -QString DatabaseObject::placePath(PlaceId place) const { - return _path + PlaceFromId(place); -} - -bool DatabaseObject::isFreePlace(PlaceId place) const { - return !QFile(placePath(place)).exists(); -} - -} // namespace details -} // namespace Cache -} // namespace Storage diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_database_object.h b/Telegram/SourceFiles/storage/cache/storage_cache_database_object.h deleted file mode 100644 index f729b985f..000000000 --- a/Telegram/SourceFiles/storage/cache/storage_cache_database_object.h +++ /dev/null @@ -1,256 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#pragma once - -#include "storage/cache/storage_cache_database.h" -#include "storage/storage_encrypted_file.h" -#include "base/binary_guard.h" -#include "base/concurrent_timer.h" -#include "base/bytes.h" -#include "base/flat_set.h" -#include -#include - -namespace Storage { -namespace Cache { -namespace details { - -class Cleaner; -class Compactor; - -class DatabaseObject { -public: - using Settings = Cache::Database::Settings; - DatabaseObject( - crl::weak_on_queue weak, - const QString &path, - const Settings &settings); - void reconfigure(const Settings &settings); - void updateSettings(const SettingsUpdate &update); - - void open(EncryptionKey &&key, FnMut &&done); - void close(FnMut &&done); - - void put( - const Key &key, - TaggedValue &&value, - FnMut &&done); - void get(const Key &key, FnMut &&done); - void remove(const Key &key, FnMut &&done); - - void putIfEmpty( - const Key &key, - TaggedValue &&value, - FnMut &&done); - void copyIfEmpty( - const Key &from, - const Key &to, - FnMut &&done); - void moveIfEmpty( - const Key &from, - const Key &to, - FnMut &&done); - - void getWithSizes( - const Key &key, - std::vector &&keys, - FnMut&&)> &&done); - - rpl::producer stats() const; - - void clear(FnMut &&done); - void clearByTag(uint8 tag, FnMut &&done); - void waitForCleaner(FnMut &&done); - - static QString BinlogFilename(); - static QString CompactReadyFilename(); - - void compactorDone(const QString &path, int64 originalReadTill); - void compactorFail(); - - struct Entry { - Entry() = default; - Entry( - PlaceId place, - uint8 tag, - uint32 checksum, - size_type size, - uint64 useTime); - - uint64 useTime = 0; - size_type size = 0; - uint32 checksum = 0; - PlaceId place = { { 0 } }; - uint8 tag = 0; - }; - using Raw = std::pair; - std::vector getManyRaw(const std::vector &keys) const; - - ~DatabaseObject(); - -private: - struct CleanerWrap { - std::unique_ptr object; - base::binary_guard guard; - FnMut done; - }; - struct CompactorWrap { - std::unique_ptr object; - int64 excessLength = 0; - crl::time nextAttempt = 0; - crl::time delayAfterFailure = 10 * crl::time(1000); - base::binary_guard guard; - }; - using Map = std::unordered_map; - - template - void invokeCallback(Callback &&callback, Args &&...args) const; - - Error ioError(const QString &path) const; - - void checkSettings(); - QString computePath(Version version) const; - QString binlogPath(Version version) const; - QString binlogPath() const; - QString compactReadyPath(Version version) const; - QString compactReadyPath() const; - Error openSomeBinlog(EncryptionKey &&key); - Error openNewBinlog(EncryptionKey &key); - File::Result openBinlog( - Version version, - File::Mode mode, - EncryptionKey &key); - bool readHeader(); - bool writeHeader(); - - void readBinlog(); - template - void readBinlogHelper(Reader &reader, Handlers &&...handlers); - template - bool processRecordStoreGeneric( - const Record *record, - Postprocess &&postprocess); - bool processRecordStore(const Store *record, std::is_class); - bool processRecordStore( - const StoreWithTime *record, - std::is_class); - template - bool processRecordMultiStore( - const Record &header, - const GetElement &element); - template - bool processRecordMultiRemove( - const MultiRemove &header, - const GetElement &element); - template - bool processRecordMultiAccess( - const MultiAccess &header, - const GetElement &element); - - void optimize(); - void checkCompactor(); - void adjustRelativeTime(); - bool startDelayedPruning(); - uint64 countRelativeTime() const; - EstimatedTimePoint countTimePoint() const; - void applyTimePoint(EstimatedTimePoint time); - - uint64 pruneBeforeTime() const; - void prune(); - void collectTimeStale( - base::flat_set &stale, - int64 &staleTotalSize); - void collectSizeStale( - base::flat_set &stale, - int64 &staleTotalSize); - void startStaleClear(); - void clearStaleNow(const base::flat_set &stale); - void clearStaleChunkDelayed(); - void clearStaleChunk(); - - void updateStats(const Entry &was, const Entry &now); - Stats collectStats() const; - void pushStatsDelayed(); - void pushStats(); - - void setMapEntry(const Key &key, Entry &&entry); - void eraseMapEntry(const Map::const_iterator &i); - void recordEntryAccess(const Key &key); - QByteArray readValueData(PlaceId place, size_type size) const; - - Version findAvailableVersion() const; - QString versionPath() const; - bool writeVersion(Version version); - Version readVersion() const; - - QString placePath(PlaceId place) const; - bool isFreePlace(PlaceId place) const; - - template - std::optional writeKeyPlaceGeneric( - StoreRecord &&record, - const Key &key, - const TaggedValue &value, - uint32 checksum); - std::optional writeKeyPlace( - const Key &key, - const TaggedValue &value, - uint32 checksum); - template - Error writeExistingPlaceGeneric( - StoreRecord &&record, - const Key &key, - const Entry &entry); - Error writeExistingPlace( - const Key &key, - const Entry &entry); - void writeMultiRemoveLazy(); - Error writeMultiRemove(); - void writeMultiAccessLazy(); - Error writeMultiAccess(); - Error writeMultiAccessBlock(); - void writeBundlesLazy(); - void writeBundles(); - - void createCleaner(); - void cleanerDone(Error error); - void clearState(); - - crl::weak_on_queue _weak; - QString _base, _path; - Settings _settings; - EncryptionKey _key; - File _binlog; - Map _map; - std::set _removing; - std::set _accessed; - std::vector _stale; - - EstimatedTimePoint _time; - - int64 _binlogExcessLength = 0; - int64 _totalSize = 0; - uint64 _minimalEntryTime = 0; - size_type _entriesWithMinimalTimeCount = 0; - - base::flat_map _taggedStats; - rpl::event_stream _stats; - bool _pushingStats = false; - bool _clearingStale = false; - - base::ConcurrentTimer _writeBundlesTimer; - base::ConcurrentTimer _pruneTimer; - - CleanerWrap _cleaner; - CompactorWrap _compactor; - -}; - -} // namespace details -} // namespace Cache -} // namespace Storage diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_database_tests.cpp b/Telegram/SourceFiles/storage/cache/storage_cache_database_tests.cpp deleted file mode 100644 index ef3f81e31..000000000 --- a/Telegram/SourceFiles/storage/cache/storage_cache_database_tests.cpp +++ /dev/null @@ -1,740 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#include "catch.hpp" - -#include "storage/cache/storage_cache_database.h" -#include "storage/storage_encryption.h" -#include "storage/storage_encrypted_file.h" -#include "base/concurrent_timer.h" -#include -#include -#include -#include - -using namespace Storage::Cache; - -const auto DisableLimitsTests = false; -const auto DisableCompactTests = false; -const auto DisableLargeTest = true; - -const auto key = Storage::EncryptionKey(bytes::make_vector( - bytes::make_span("\ -abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\ -abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\ -abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\ -abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\ -").subspan(0, Storage::EncryptionKey::kSize))); - -const auto name = QString("test.db"); - -const auto SmallSleep = [] { - static auto SleepTime = 0; - if (SleepTime > 5000) { - return false; - } - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - SleepTime += 10; - return true; -}; - -QString GetBinlogPath() { - using namespace Storage; - - QFile versionFile(name + "/version"); - while (!versionFile.open(QIODevice::ReadOnly)) { - if (!SmallSleep()) { - return QString(); - } - } - const auto bytes = versionFile.readAll(); - if (bytes.size() != 4) { - return QString(); - } - const auto version = *reinterpret_cast(bytes.data()); - return name + '/' + QString::number(version) + "/binlog"; -} - -const auto Test1 = [] { - static auto result = QByteArray("testbytetestbyt"); - return result; -}; -const auto Test2 = [] { - static auto result = QByteArray("bytetestbytetestb"); - return result; -}; - -crl::semaphore Semaphore; - -auto Result = Error(); -const auto GetResult = [](Error error) { - Result = error; - Semaphore.release(); -}; - -auto Value = QByteArray(); -const auto GetValue = [](QByteArray value) { - Value = value; - Semaphore.release(); -}; - -auto ValueWithTag = Database::TaggedValue(); -const auto GetValueWithTag = [](Database::TaggedValue value) { - ValueWithTag = value; - Semaphore.release(); -}; - -Error Open(Database &db, const Storage::EncryptionKey &key) { - db.open(base::duplicate(key), GetResult); - Semaphore.acquire(); - return Result; -} - -void Close(Database &db) { - db.close([&] { Semaphore.release(); }); - Semaphore.acquire(); -} - -Error Clear(Database &db) { - db.clear(GetResult); - Semaphore.acquire(); - return Result; -} - -QByteArray Get(Database &db, const Key &key) { - db.get(key, GetValue); - Semaphore.acquire(); - return Value; -} - -Database::TaggedValue GetWithTag(Database &db, const Key &key) { - db.getWithTag(key, GetValueWithTag); - Semaphore.acquire(); - return ValueWithTag; -} - -Error Put(Database &db, const Key &key, QByteArray &&value) { - db.put(key, std::move(value), GetResult); - Semaphore.acquire(); - return Result; -} - -Error Put(Database &db, const Key &key, Database::TaggedValue &&value) { - db.put(key, std::move(value), GetResult); - Semaphore.acquire(); - return Result; -} - -Error PutIfEmpty(Database &db, const Key &key, QByteArray &&value) { - db.putIfEmpty(key, std::move(value), GetResult); - Semaphore.acquire(); - return Result; -} - -Error CopyIfEmpty(Database &db, const Key &from, const Key &to) { - db.copyIfEmpty(from, to, GetResult); - Semaphore.acquire(); - return Result; -} - -Error MoveIfEmpty(Database &db, const Key &from, const Key &to) { - db.moveIfEmpty(from, to, GetResult); - Semaphore.acquire(); - return Result; -} - -void Remove(Database &db, const Key &key) { - db.remove(key, [&](Error) { Semaphore.release(); }); - Semaphore.acquire(); -} - -Error ClearByTag(Database &db, uint8 tag) { - db.clearByTag(tag, GetResult); - Semaphore.acquire(); - return Result; -} - -const auto Settings = [] { - auto result = Database::Settings(); - result.trackEstimatedTime = false; - result.writeBundleDelay = 1 * crl::time(1000); - result.pruneTimeout = 1 * crl::time(1500); - result.maxDataSize = 20; - return result; -}(); - -const auto AdvanceTime = [](int32 seconds) { - std::this_thread::sleep_for(std::chrono::milliseconds(1000) * seconds); -}; - -TEST_CASE("init timers", "[storage_cache_database]") { - static auto init = [] { - int argc = 0; - char **argv = nullptr; - static QCoreApplication application(argc, argv); - static base::ConcurrentTimerEnvironment environment; - return true; - }(); -} - -TEST_CASE("compacting db", "[storage_cache_database]") { - if (DisableCompactTests || !DisableLargeTest) { - return; - } - const auto write = [](Database &db, uint32 from, uint32 till, QByteArray base) { - for (auto i = from; i != till; ++i) { - auto value = base; - value[0] = char('A') + i; - const auto result = Put(db, Key{ i, i + 1 }, std::move(value)); - REQUIRE(result.type == Error::Type::None); - } - }; - const auto put = [&](Database &db, uint32 from, uint32 till) { - write(db, from, till, Test1()); - }; - const auto reput = [&](Database &db, uint32 from, uint32 till) { - write(db, from, till, Test2()); - }; - const auto remove = [](Database &db, uint32 from, uint32 till) { - for (auto i = from; i != till; ++i) { - Remove(db, Key{ i, i + 1 }); - } - }; - const auto get = [](Database &db, uint32 from, uint32 till) { - for (auto i = from; i != till; ++i) { - db.get(Key{ i, i + 1 }, nullptr); - } - }; - const auto check = [](Database &db, uint32 from, uint32 till, QByteArray base) { - for (auto i = from; i != till; ++i) { - auto value = base; - if (!value.isEmpty()) { - value[0] = char('A') + i; - } - const auto result = Get(db, Key{ i, i + 1 }); - REQUIRE((result == value)); - } - }; - SECTION("simple compact with min size") { - auto settings = Settings; - settings.writeBundleDelay = crl::time(100); - settings.readBlockSize = 512; - settings.maxBundledRecords = 5; - settings.compactAfterExcess = (3 * (16 * 5 + 16) + 15 * 32) / 2; - settings.compactAfterFullSize = (sizeof(details::BasicHeader) - + 40 * 32) / 2 - + settings.compactAfterExcess; - Database db(name, settings); - - REQUIRE(Clear(db).type == Error::Type::None); - REQUIRE(Open(db, key).type == Error::Type::None); - put(db, 0, 30); - remove(db, 0, 15); - put(db, 30, 40); - reput(db, 15, 29); - AdvanceTime(1); - const auto path = GetBinlogPath(); - const auto size = QFile(path).size(); - reput(db, 29, 30); // starts compactor - AdvanceTime(2); - REQUIRE(QFile(path).size() < size); - remove(db, 30, 35); - reput(db, 35, 37); - put(db, 15, 20); - put(db, 40, 45); - - const auto fullcheck = [&] { - check(db, 0, 15, {}); - check(db, 15, 20, Test1()); - check(db, 20, 30, Test2()); - check(db, 30, 35, {}); - check(db, 35, 37, Test2()); - check(db, 37, 45, Test1()); - }; - fullcheck(); - Close(db); - - REQUIRE(Open(db, key).type == Error::Type::None); - fullcheck(); - Close(db); - } - SECTION("simple compact without min size") { - auto settings = Settings; - settings.writeBundleDelay = crl::time(100); - settings.readBlockSize = 512; - settings.maxBundledRecords = 5; - settings.compactAfterExcess = 3 * (16 * 5 + 16) + 15 * 32; - Database db(name, settings); - - REQUIRE(Clear(db).type == Error::Type::None); - REQUIRE(Open(db, key).type == Error::Type::None); - put(db, 0, 30); - remove(db, 0, 15); - put(db, 30, 40); - reput(db, 15, 29); - AdvanceTime(1); - const auto path = GetBinlogPath(); - const auto size = QFile(path).size(); - reput(db, 29, 30); // starts compactor - AdvanceTime(2); - REQUIRE(QFile(path).size() < size); - remove(db, 30, 35); - reput(db, 35, 37); - put(db, 15, 20); - put(db, 40, 45); - - const auto fullcheck = [&] { - check(db, 0, 15, {}); - check(db, 15, 20, Test1()); - check(db, 20, 30, Test2()); - check(db, 30, 35, {}); - check(db, 35, 37, Test2()); - check(db, 37, 45, Test1()); - }; - fullcheck(); - Close(db); - - REQUIRE(Open(db, key).type == Error::Type::None); - fullcheck(); - Close(db); - } - SECTION("double compact") { - auto settings = Settings; - settings.writeBundleDelay = crl::time(100); - settings.readBlockSize = 512; - settings.maxBundledRecords = 5; - settings.compactAfterExcess = 3 * (16 * 5 + 16) + 15 * 32; - Database db(name, settings); - - REQUIRE(Clear(db).type == Error::Type::None); - REQUIRE(Open(db, key).type == Error::Type::None); - put(db, 0, 30); - remove(db, 0, 15); - reput(db, 15, 29); - AdvanceTime(1); - const auto path = GetBinlogPath(); - const auto size1 = QFile(path).size(); - reput(db, 29, 30); // starts compactor - AdvanceTime(2); - REQUIRE(QFile(path).size() < size1); - put(db, 30, 45); - remove(db, 20, 35); - put(db, 15, 20); - reput(db, 35, 44); - const auto size2 = QFile(path).size(); - reput(db, 44, 45); // starts compactor - AdvanceTime(2); - const auto after = QFile(path).size(); - REQUIRE(after < size1); - REQUIRE(after < size2); - const auto fullcheck = [&] { - check(db, 0, 15, {}); - check(db, 15, 20, Test1()); - check(db, 20, 35, {}); - check(db, 35, 45, Test2()); - }; - fullcheck(); - Close(db); - - REQUIRE(Open(db, key).type == Error::Type::None); - fullcheck(); - Close(db); - } - SECTION("time tracking compact") { - auto settings = Settings; - settings.writeBundleDelay = crl::time(100); - settings.trackEstimatedTime = true; - settings.readBlockSize = 512; - settings.maxBundledRecords = 5; - settings.compactAfterExcess = 6 * (16 * 5 + 16) - + 3 * (16 * 5 + 16) - + 15 * 48 - + 3 * (16 * 5 + 16) - + (16 * 1 + 16); - Database db(name, settings); - - REQUIRE(Clear(db).type == Error::Type::None); - REQUIRE(Open(db, key).type == Error::Type::None); - put(db, 0, 30); - get(db, 0, 30); - //AdvanceTime(1); get's will be written instantly becase !(30 % 5) - remove(db, 0, 15); - reput(db, 15, 30); - get(db, 0, 30); - AdvanceTime(1); - const auto path = GetBinlogPath(); - const auto size = QFile(path).size(); - get(db, 29, 30); // starts compactor delayed - AdvanceTime(2); - REQUIRE(QFile(path).size() < size); - const auto fullcheck = [&] { - check(db, 15, 30, Test2()); - }; - fullcheck(); - Close(db); - - REQUIRE(Open(db, key).type == Error::Type::None); - fullcheck(); - Close(db); - } -} - -TEST_CASE("encrypted cache db", "[storage_cache_database]") { - if (!DisableLargeTest) { - return; - } - SECTION("writing db") { - Database db(name, Settings); - - REQUIRE(Clear(db).type == Error::Type::None); - REQUIRE(Open(db, key).type == Error::Type::None); - REQUIRE(Put(db, Key{ 0, 1 }, Test2()).type == Error::Type::None); - REQUIRE(Put(db, Key{ 0, 1 }, Database::TaggedValue(Test1(), 1)).type - == Error::Type::None); - REQUIRE(PutIfEmpty(db, Key{ 0, 2 }, Test2()).type - == Error::Type::None); - REQUIRE(PutIfEmpty(db, Key{ 0, 2 }, Test1()).type - == Error::Type::None); - REQUIRE(CopyIfEmpty(db, Key{ 0, 1 }, Key{ 2, 0 }).type - == Error::Type::None); - REQUIRE(CopyIfEmpty(db, Key{ 0, 2 }, Key{ 2, 0 }).type - == Error::Type::None); - REQUIRE(Put(db, Key{ 0, 3 }, Test1()).type == Error::Type::None); - REQUIRE(MoveIfEmpty(db, Key{ 0, 3 }, Key{ 3, 0 }).type - == Error::Type::None); - REQUIRE(MoveIfEmpty(db, Key{ 0, 2 }, Key{ 3, 0 }).type - == Error::Type::None); - Close(db); - } - SECTION("reading and writing db") { - Database db(name, Settings); - - REQUIRE(Open(db, key).type == Error::Type::None); - REQUIRE((Get(db, Key{ 0, 1 }) == Test1())); - const auto withTag1 = GetWithTag(db, Key{ 0, 1 }); - REQUIRE(((withTag1.bytes == Test1()) && (withTag1.tag == 1))); - REQUIRE(Put(db, Key{ 1, 0 }, Test2()).type == Error::Type::None); - const auto withTag2 = GetWithTag(db, Key{ 1, 0 }); - REQUIRE(((withTag2.bytes == Test2()) && (withTag2.tag == 0))); - REQUIRE(Get(db, Key{ 1, 1 }).isEmpty()); - REQUIRE((Get(db, Key{ 0, 2 }) == Test2())); - REQUIRE((Get(db, Key{ 2, 0 }) == Test1())); - REQUIRE(Get(db, Key{ 0, 3 }).isEmpty()); - REQUIRE((Get(db, Key{ 3, 0 }) == Test1())); - - REQUIRE(Put(db, Key{ 5, 1 }, Database::TaggedValue(Test1(), 1)).type - == Error::Type::None); - REQUIRE(Put(db, Key{ 6, 1 }, Database::TaggedValue(Test2(), 1)).type - == Error::Type::None); - REQUIRE(Put(db, Key{ 5, 2 }, Database::TaggedValue(Test1(), 2)).type - == Error::Type::None); - REQUIRE(Put(db, Key{ 6, 2 }, Database::TaggedValue(Test2(), 2)).type - == Error::Type::None); - REQUIRE(Put(db, Key{ 5, 3 }, Database::TaggedValue(Test1(), 3)).type - == Error::Type::None); - REQUIRE(Put(db, Key{ 6, 3 }, Database::TaggedValue(Test2(), 3)).type - == Error::Type::None); - Close(db); - } - SECTION("reading db") { - Database db(name, Settings); - - REQUIRE(Open(db, key).type == Error::Type::None); - REQUIRE((Get(db, Key{ 0, 1 }) == Test1())); - REQUIRE((Get(db, Key{ 1, 0 }) == Test2())); - Close(db); - } - SECTION("deleting in db by tag") { - Database db(name, Settings); - - REQUIRE(Open(db, key).type == Error::Type::None); - REQUIRE(ClearByTag(db, 2).type == Error::Type::None); - REQUIRE((Get(db, Key{ 1, 0 }) == Test2())); - - const auto withTag1 = GetWithTag(db, Key{ 5, 1 }); - REQUIRE(((withTag1.bytes == Test1()) && (withTag1.tag == 1))); - const auto withTag2 = GetWithTag(db, Key{ 6, 1 }); - REQUIRE(((withTag2.bytes == Test2()) && (withTag2.tag == 1))); - REQUIRE(Get(db, Key{ 5, 2 }).isEmpty()); - REQUIRE(Get(db, Key{ 6, 2 }).isEmpty()); - const auto withTag3 = GetWithTag(db, Key{ 5, 3 }); - REQUIRE(((withTag3.bytes == Test1()) && (withTag3.tag == 3))); - const auto withTag4 = GetWithTag(db, Key{ 6, 3 }); - REQUIRE(((withTag4.bytes == Test2()) && (withTag4.tag == 3))); - Close(db); - } - SECTION("overwriting values") { - Database db(name, Settings); - - REQUIRE(Open(db, key).type == Error::Type::None); - const auto path = GetBinlogPath(); - REQUIRE((Get(db, Key{ 0, 1 }) == Test1())); - const auto size = QFile(path).size(); - REQUIRE(Put(db, Key{ 0, 1 }, Test2()).type == Error::Type::None); - const auto next = QFile(path).size(); - REQUIRE(next > size); - REQUIRE((Get(db, Key{ 0, 1 }) == Test2())); - REQUIRE(Put(db, Key{ 0, 1 }, Test2()).type == Error::Type::None); - const auto same = QFile(path).size(); - REQUIRE(same == next); - Close(db); - } - SECTION("reading db in many chunks") { - auto settings = Settings; - settings.readBlockSize = 512; - settings.maxBundledRecords = 5; - settings.trackEstimatedTime = true; - Database db(name, settings); - - const auto count = 30U; - - REQUIRE(Clear(db).type == Error::Type::None); - REQUIRE(Open(db, key).type == Error::Type::None); - for (auto i = 0U; i != count; ++i) { - auto value = Test1(); - value[0] = char('A') + i; - const auto result = Put(db, Key{ i, i * 2 }, std::move(value)); - REQUIRE(result.type == Error::Type::None); - } - Close(db); - - REQUIRE(Open(db, key).type == Error::Type::None); - for (auto i = 0U; i != count; ++i) { - auto value = Test1(); - value[0] = char('A') + i; - REQUIRE((Get(db, Key{ i, i * 2 }) == value)); - } - Close(db); - } -} - -TEST_CASE("cache db remove", "[storage_cache_database]") { - if (!DisableLargeTest) { - return; - } - SECTION("db remove deletes value") { - Database db(name, Settings); - - REQUIRE(Clear(db).type == Error::Type::None); - REQUIRE(Open(db, key).type == Error::Type::None); - REQUIRE(Put(db, Key{ 0, 1 }, Test1()).type == Error::Type::None); - REQUIRE(Put(db, Key{ 1, 0 }, Test2()).type == Error::Type::None); - Remove(db, Key{ 0, 1 }); - REQUIRE(Get(db, Key{ 0, 1 }).isEmpty()); - REQUIRE((Get(db, Key{ 1, 0 }) == Test2())); - Close(db); - } - SECTION("db remove deletes value permanently") { - Database db(name, Settings); - - REQUIRE(Open(db, key).type == Error::Type::None); - REQUIRE(Get(db, Key{ 0, 1 }).isEmpty()); - REQUIRE((Get(db, Key{ 1, 0 }) == Test2())); - Close(db); - } -} - -TEST_CASE("cache db bundled actions", "[storage_cache_database]") { - if (!DisableLargeTest) { - return; - } - SECTION("db touched written lazily") { - auto settings = Settings; - settings.trackEstimatedTime = true; - Database db(name, settings); - - REQUIRE(Clear(db).type == Error::Type::None); - REQUIRE(Open(db, key).type == Error::Type::None); - const auto path = GetBinlogPath(); - REQUIRE(Put(db, Key{ 0, 1 }, Test1()).type == Error::Type::None); - const auto size = QFile(path).size(); - REQUIRE((Get(db, Key{ 0, 1 }) == Test1())); - REQUIRE(QFile(path).size() == size); - AdvanceTime(2); - Get(db, Key{ 0, 1 }); - REQUIRE(QFile(path).size() > size); - Close(db); - } - SECTION("db touched written on close") { - auto settings = Settings; - settings.trackEstimatedTime = true; - Database db(name, settings); - - REQUIRE(Clear(db).type == Error::Type::None); - REQUIRE(Open(db, key).type == Error::Type::None); - const auto path = GetBinlogPath(); - REQUIRE(Put(db, Key{ 0, 1 }, Test1()).type == Error::Type::None); - const auto size = QFile(path).size(); - REQUIRE((Get(db, Key{ 0, 1 }) == Test1())); - REQUIRE(QFile(path).size() == size); - Close(db); - REQUIRE(QFile(path).size() > size); - } - SECTION("db remove written lazily") { - Database db(name, Settings); - - REQUIRE(Clear(db).type == Error::Type::None); - REQUIRE(Open(db, key).type == Error::Type::None); - const auto path = GetBinlogPath(); - REQUIRE(Put(db, Key{ 0, 1 }, Test1()).type == Error::Type::None); - const auto size = QFile(path).size(); - Remove(db, Key{ 0, 1 }); - REQUIRE(QFile(path).size() == size); - AdvanceTime(2); - REQUIRE(QFile(path).size() > size); - Close(db); - } - SECTION("db remove written on close") { - Database db(name, Settings); - - REQUIRE(Clear(db).type == Error::Type::None); - REQUIRE(Open(db, key).type == Error::Type::None); - const auto path = GetBinlogPath(); - REQUIRE(Put(db, Key{ 0, 1 }, Test1()).type == Error::Type::None); - const auto size = QFile(path).size(); - Remove(db, Key{ 0, 1 }); - REQUIRE(QFile(path).size() == size); - Close(db); - REQUIRE(QFile(path).size() > size); - } -} - -TEST_CASE("cache db limits", "[storage_cache_database]") { - if (DisableLimitsTests || !DisableLargeTest) { - return; - } - SECTION("db both limit") { - auto settings = Settings; - settings.trackEstimatedTime = true; - settings.totalSizeLimit = 17 * 3 + 1; - settings.totalTimeLimit = 4; - Database db(name, settings); - - db.clear(nullptr); - db.open(base::duplicate(key), nullptr); - db.put(Key{ 0, 1 }, Test1(), nullptr); - db.put(Key{ 1, 0 }, Test2(), nullptr); - AdvanceTime(2); - db.get(Key{ 1, 0 }, nullptr); - AdvanceTime(3); - db.put(Key{ 1, 1 }, Test1(), nullptr); - db.put(Key{ 2, 0 }, Test2(), nullptr); - db.put(Key{ 0, 2 }, Test1(), nullptr); - AdvanceTime(2); - REQUIRE(Get(db, Key{ 0, 1 }).isEmpty()); - REQUIRE(Get(db, Key{ 1, 0 }).isEmpty()); - REQUIRE((Get(db, Key{ 1, 1 }) == Test1())); - REQUIRE((Get(db, Key{ 2, 0 }) == Test2())); - REQUIRE((Get(db, Key{ 0, 2 }) == Test1())); - Close(db); - } - SECTION("db size limit") { - auto settings = Settings; - settings.trackEstimatedTime = true; - settings.totalSizeLimit = 17 * 3 + 1; - Database db(name, settings); - - db.clear(nullptr); - db.open(base::duplicate(key), nullptr); - db.put(Key{ 0, 1 }, Test1(), nullptr); - AdvanceTime(2); - db.put(Key{ 1, 0 }, Test2(), nullptr); - AdvanceTime(2); - db.put(Key{ 1, 1 }, Test1(), nullptr); - db.get(Key{ 0, 1 }, nullptr); - AdvanceTime(2); - db.put(Key{ 2, 0 }, Test2(), nullptr); - - // Removing { 1, 0 } will be scheduled. - REQUIRE((Get(db, Key{ 0, 1 }) == Test1())); - REQUIRE((Get(db, Key{ 1, 1 }) == Test1())); - REQUIRE((Get(db, Key{ 2, 0 }) == Test2())); - AdvanceTime(2); - - // Removing { 1, 0 } performed. - REQUIRE(Get(db, Key{ 1, 0 }).isEmpty()); - REQUIRE((Get(db, Key{ 1, 1 }) == Test1())); - db.put(Key{ 0, 2 }, Test1(), nullptr); - REQUIRE(Put(db, Key{ 2, 2 }, Test2()).type == Error::Type::None); - - // Removing { 0, 1 } and { 2, 0 } will be scheduled. - AdvanceTime(2); - - // Removing { 0, 1 } and { 2, 0 } performed. - REQUIRE(Get(db, Key{ 0, 1 }).isEmpty()); - REQUIRE(Get(db, Key{ 2, 0 }).isEmpty()); - REQUIRE((Get(db, Key{ 1, 1 }) == Test1())); - REQUIRE((Get(db, Key{ 0, 2 }) == Test1())); - REQUIRE((Get(db, Key{ 2, 2 }) == Test2())); - Close(db); - } - SECTION("db time limit") { - auto settings = Settings; - settings.trackEstimatedTime = true; - settings.totalTimeLimit = 3; - Database db(name, settings); - - db.clear(nullptr); - db.open(base::duplicate(key), nullptr); - db.put(Key{ 0, 1 }, Test1(), nullptr); - db.put(Key{ 1, 0 }, Test2(), nullptr); - db.put(Key{ 1, 1 }, Test1(), nullptr); - db.put(Key{ 2, 0 }, Test2(), nullptr); - AdvanceTime(1); - db.get(Key{ 1, 0 }, nullptr); - db.get(Key{ 1, 1 }, nullptr); - AdvanceTime(1); - db.get(Key{ 1, 0 }, nullptr); - db.get(Key{ 0, 1 }, nullptr); - AdvanceTime(1); - db.get(Key{ 1, 0 }, nullptr); - db.get(Key{ 0, 1 }, nullptr); - AdvanceTime(3); - REQUIRE(Get(db, Key{ 2, 0 }).isEmpty()); - REQUIRE(Get(db, Key{ 1, 1 }).isEmpty()); - REQUIRE((Get(db, Key{ 1, 0 }) == Test2())); - REQUIRE((Get(db, Key{ 0, 1 }) == Test1())); - Close(db); - } -} - -TEST_CASE("large db", "[storage_cache_database]") { - if (DisableLargeTest) { - return; - } - SECTION("time tracking large db") { - auto settings = Database::Settings(); - settings.writeBundleDelay = crl::time(1000); - settings.maxDataSize = 20; - settings.totalSizeLimit = 1024 * 1024; - settings.totalTimeLimit = 120; - settings.pruneTimeout = crl::time(1500); - settings.compactAfterExcess = 1024 * 1024; - settings.trackEstimatedTime = true; - Database db(name, settings); - - //REQUIRE(Clear(db).type == Error::Type::None); - REQUIRE(Open(db, key).type == Error::Type::None); - - const auto key = [](int index) { - return Key{ uint64(index) * 2, (uint64(index) << 32) + 3 }; - }; - const auto kWriteRecords = 100 * 1024; - for (auto i = 0; i != kWriteRecords; ++i) { - db.put(key(i), Test1(), nullptr); - const auto j = i ? (rand() % i) : 0; - if (i % 1024 == 1023) { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - Get(db, key(j)); - } else { - db.get(key(j), nullptr); - } - } - - Close(db); - } -} diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_types.cpp b/Telegram/SourceFiles/storage/cache/storage_cache_types.cpp deleted file mode 100644 index 7bec686bd..000000000 --- a/Telegram/SourceFiles/storage/cache/storage_cache_types.cpp +++ /dev/null @@ -1,138 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#include "storage/cache/storage_cache_types.h" - -#include - -namespace Storage { -namespace Cache { -namespace details { -namespace { - -template -inline Packed ReadTo(size_type count) { - Expects(count >= 0 && count < (1 << (Packed().size() * 8))); - - auto result = Packed(); - for (auto &element : result) { - element = uint8(count & 0xFF); - count >>= 8; - } - return result; -} - -template -inline size_type ReadFrom(const Packed &count) { - auto result = size_type(); - for (auto &element : (count | ranges::view::reverse)) { - result <<= 8; - result |= size_type(element); - } - return result; -} - -template -inline size_type ValidateStrictCount(const Packed &count) { - const auto result = ReadFrom(count); - return (result != 0) ? result : -1; -} - -} // namespace - -TaggedValue::TaggedValue(QByteArray &&bytes, uint8 tag) -: bytes(std::move(bytes)), tag(tag) { -} - -QString ComputeBasePath(const QString &original) { - const auto result = QDir(original).absolutePath(); - return result.endsWith('/') ? result : (result + '/'); -} - -QString VersionFilePath(const QString &base) { - Expects(base.endsWith('/')); - - return base + QStringLiteral("version"); -} - -std::optional ReadVersionValue(const QString &base) { - QFile file(VersionFilePath(base)); - if (!file.open(QIODevice::ReadOnly)) { - return std::nullopt; - } - const auto bytes = file.read(sizeof(Version)); - if (bytes.size() != sizeof(Version)) { - return std::nullopt; - } - return *reinterpret_cast(bytes.data()); -} - -bool WriteVersionValue(const QString &base, Version value) { - if (!QDir().mkpath(base)) { - return false; - } - const auto bytes = QByteArray::fromRawData( - reinterpret_cast(&value), - sizeof(value)); - QFile file(VersionFilePath(base)); - if (!file.open(QIODevice::WriteOnly)) { - return false; - } else if (file.write(bytes) != bytes.size()) { - return false; - } - return file.flush(); -} - -BasicHeader::BasicHeader() -: format(static_cast(Format::Format_0)) -, flags(0) { -} - -void Store::setSize(size_type size) { - this->size = ReadTo(size); -} - -size_type Store::getSize() const { - return ReadFrom(size); -} - -MultiStore::MultiStore(size_type count) -: type(kType) -, count(ReadTo(count)) { - Expects(count >= 0 && count < kBundledRecordsLimit); -} - -size_type MultiStore::validateCount() const { - return ValidateStrictCount(count); -} - -MultiRemove::MultiRemove(size_type count) -: type(kType) -, count(ReadTo(count)) { - Expects(count >= 0 && count < kBundledRecordsLimit); -} - -size_type MultiRemove::validateCount() const { - return ValidateStrictCount(count); -} - -MultiAccess::MultiAccess( - EstimatedTimePoint time, - size_type count) -: type(kType) -, count(ReadTo(count)) -, time(time) { - Expects(count >= 0 && count < kBundledRecordsLimit); -} - -size_type MultiAccess::validateCount() const { - return ReadFrom(count); -} - -} // namespace details -} // namespace Cache -} // namespace Storage diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_types.h b/Telegram/SourceFiles/storage/cache/storage_cache_types.h deleted file mode 100644 index d9e1739fc..000000000 --- a/Telegram/SourceFiles/storage/cache/storage_cache_types.h +++ /dev/null @@ -1,239 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#pragma once - -#include "base/basic_types.h" -#include "base/flat_map.h" -#include "base/optional.h" -#include -#include -#include - -namespace Storage { -namespace Cache { - -struct Key { - uint64 high = 0; - uint64 low = 0; -}; - -inline bool operator==(const Key &a, const Key &b) { - return (a.high == b.high) && (a.low == b.low); -} - -inline bool operator!=(const Key &a, const Key &b) { - return !(a == b); -} - -inline bool operator<(const Key &a, const Key &b) { - return std::tie(a.high, a.low) < std::tie(b.high, b.low); -} - -struct Error { - enum class Type { - None, - IO, - WrongKey, - LockFailed, - }; - Type type = Type::None; - QString path; - - static Error NoError(); -}; - -inline Error Error::NoError() { - return Error(); -} - -namespace details { - -using RecordType = uint8; -using PlaceId = std::array; -using EntrySize = std::array; -using RecordsCount = std::array; - -constexpr auto kRecordSizeUnknown = size_type(-1); -constexpr auto kRecordSizeInvalid = size_type(-2); -constexpr auto kBundledRecordsLimit - = size_type(1 << (RecordsCount().size() * 8)); -constexpr auto kDataSizeLimit = size_type(1 << (EntrySize().size() * 8)); - -struct Settings { - size_type maxBundledRecords = 16 * 1024; - size_type readBlockSize = 8 * 1024 * 1024; - size_type maxDataSize = (kDataSizeLimit - 1); - crl::time writeBundleDelay = 15 * 60 * crl::time(1000); - size_type staleRemoveChunk = 256; - - int64 compactAfterExcess = 8 * 1024 * 1024; - int64 compactAfterFullSize = 0; - size_type compactChunkSize = 16 * 1024; - - bool trackEstimatedTime = true; - int64 totalSizeLimit = 1024 * 1024 * 1024; - size_type totalTimeLimit = 31 * 24 * 60 * 60; // One month in seconds. - crl::time pruneTimeout = 5 * crl::time(1000); - crl::time maxPruneCheckTimeout = 3600 * crl::time(1000); - - bool clearOnWrongKey = false; -}; - -struct SettingsUpdate { - int64 totalSizeLimit = Settings().totalSizeLimit; - size_type totalTimeLimit = Settings().totalTimeLimit; -}; - -struct TaggedValue { - TaggedValue() = default; - TaggedValue(QByteArray &&bytes, uint8 tag); - - QByteArray bytes; - uint8 tag = 0; -}; - -struct TaggedSummary { - size_type count = 0; - int64 totalSize = 0; -}; -struct Stats { - TaggedSummary full; - base::flat_map tagged; - bool clearing = false; -}; - -using Version = int32; - -QString ComputeBasePath(const QString &original); -QString VersionFilePath(const QString &base); -std::optional ReadVersionValue(const QString &base); -bool WriteVersionValue(const QString &base, Version value); - -template -constexpr auto GoodForEncryption = ((sizeof(Record) & 0x0F) == 0); - -enum class Format : uint32 { - Format_0, -}; - -struct BasicHeader { - BasicHeader(); - - static constexpr auto kTrackEstimatedTime = 0x01U; - - Format getFormat() const { - return static_cast(format); - } - void setFormat(Format format) { - this->format = static_cast(format); - } - - uint32 format : 8; - uint32 flags : 24; - uint32 systemTime = 0; - uint32 reserved1 = 0; - uint32 reserved2 = 0; -}; - -struct EstimatedTimePoint { - uint32 relative1 = 0; - uint32 relative2 = 0; - uint32 system = 0; - - void setRelative(uint64 value) { - relative1 = uint32(value & 0xFFFFFFFFU); - relative2 = uint32((value >> 32) & 0xFFFFFFFFU); - } - uint64 getRelative() const { - return uint64(relative1) | (uint64(relative2) << 32); - } -}; - -struct Store { - static constexpr auto kType = RecordType(0x01); - - void setSize(size_type size); - size_type getSize() const; - - RecordType type = kType; - uint8 tag = 0; - EntrySize size = { { 0 } }; - PlaceId place = { { 0 } }; - uint32 checksum = 0; - Key key; -}; - -struct StoreWithTime : Store { - EstimatedTimePoint time; - uint32 reserved = 0; -}; - -struct MultiStore { - static constexpr auto kType = RecordType(0x02); - - explicit MultiStore(size_type count = 0); - - RecordType type = kType; - RecordsCount count = { { 0 } }; - uint32 reserved1 = 0; - uint32 reserved2 = 0; - uint32 reserved3 = 0; - - using Part = Store; - size_type validateCount() const; -}; -struct MultiStoreWithTime : MultiStore { - using MultiStore::MultiStore; - - using Part = StoreWithTime; -}; - -struct MultiRemove { - static constexpr auto kType = RecordType(0x03); - - explicit MultiRemove(size_type count = 0); - - RecordType type = kType; - RecordsCount count = { { 0 } }; - uint32 reserved1 = 0; - uint32 reserved2 = 0; - uint32 reserved3 = 0; - - using Part = Key; - size_type validateCount() const; -}; - -struct MultiAccess { - static constexpr auto kType = RecordType(0x04); - - explicit MultiAccess( - EstimatedTimePoint time, - size_type count = 0); - - RecordType type = kType; - RecordsCount count = { { 0 } }; - EstimatedTimePoint time; - - using Part = Key; - size_type validateCount() const; -}; - -} // namespace details -} // namespace Cache -} // namespace Storage - -namespace std { - -template <> -struct hash { - size_t operator()(const Storage::Cache::Key &key) const { - return (hash()(key.high) ^ hash()(key.low)); - } -}; - -} // namespace std diff --git a/Telegram/SourceFiles/storage/storage_clear_legacy.cpp b/Telegram/SourceFiles/storage/storage_clear_legacy.cpp deleted file mode 100644 index 993d79d63..000000000 --- a/Telegram/SourceFiles/storage/storage_clear_legacy.cpp +++ /dev/null @@ -1,53 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#include "storage/storage_clear_legacy.h" - -#include - -namespace Storage { -namespace { - -constexpr auto kClearPartSize = size_type(10000); - -} // namespace - -void ClearLegacyFilesPart( - const QString &base, - CollectGoodFiles filter, - base::flat_set &&skip = {}) { - filter([ - =, - files = details::CollectFiles(base, kClearPartSize, skip) - ](base::flat_set &&skip) mutable { - crl::async([ - =, - files = std::move(files), - skip = std::move(skip) - ]() mutable { - for (const auto &name : files) { - if (!skip.contains(name) - && !details::RemoveLegacyFile(base + name)) { - skip.emplace(name); - } - } - if (files.size() == kClearPartSize) { - ClearLegacyFilesPart(base, filter, std::move(skip)); - } - }); - }); -} - -void ClearLegacyFiles(const QString &base, CollectGoodFiles filter) { - Expects(base.endsWith('/')); - - crl::async([=] { - ClearLegacyFilesPart(base, std::move(filter)); - }); -} - -} // namespace Storage diff --git a/Telegram/SourceFiles/storage/storage_clear_legacy.h b/Telegram/SourceFiles/storage/storage_clear_legacy.h deleted file mode 100644 index 09ee4f959..000000000 --- a/Telegram/SourceFiles/storage/storage_clear_legacy.h +++ /dev/null @@ -1,26 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#pragma once - -namespace Storage { - -using CollectGoodFiles = Fn&&)>)>; - -void ClearLegacyFiles(const QString &base, CollectGoodFiles filter); - -namespace details { - -std::vector CollectFiles( - const QString &base, - size_type limit, - const base::flat_set &skip); - -bool RemoveLegacyFile(const QString &path); - -} // namespace details -} // namespace Storage diff --git a/Telegram/SourceFiles/storage/storage_clear_legacy_posix.cpp b/Telegram/SourceFiles/storage/storage_clear_legacy_posix.cpp deleted file mode 100644 index bb92db8f3..000000000 --- a/Telegram/SourceFiles/storage/storage_clear_legacy_posix.cpp +++ /dev/null @@ -1,101 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#include "storage/storage_clear_legacy.h" - -#include -#include -#include - -namespace Storage { -namespace details { - -std::vector CollectFiles( - const QString &base, - size_type limit, - const base::flat_set &skip) { - Expects(base.endsWith('/')); - Expects(limit > 0); - - const auto path = QFile::encodeName(base); - const auto folder = path.mid(0, path.size() - 1); - const auto directory = opendir(folder.constData()); - if (!directory) { - return {}; - } - const auto guard = gsl::finally([&] { closedir(directory); }); - - auto result = std::vector(); - while (const auto entry = readdir(directory)) { - const auto local = entry->d_name; - if (!strcmp(local, ".") || !strcmp(local, "..")) { - continue; - } - - const auto full = path + QByteArray(local); - const auto data = full.constData(); - struct stat statbuf = { 0 }; - if (stat(full.constData(), &statbuf) != 0 || S_ISDIR(statbuf.st_mode)) { - continue; - } - - auto name = QFile::decodeName(local); - if (!skip.contains(name)) { - result.push_back(std::move(name)); - } - if (result.size() == limit) { - break; - } - } - return result; - -// // It looks like POSIX solution works fine on macOS so no need for Cocoa solution. -// -// NSString *native = [NSString stringWithUTF8String:utf8.constData()]; -// NSFileManager *manager = [NSFileManager defaultManager]; -// NSArray *properties = [NSArray arrayWithObject:NSURLIsDirectoryKey]; -// NSDirectoryEnumerator *enumerator = [manager -// enumeratorAtURL:[NSURL fileURLWithPath:native] -// includingPropertiesForKeys:properties -// options:0 -// errorHandler:^(NSURL *url, NSError *error) { -// return NO; -// }]; -// -// auto result = std::vector(); -// for (NSURL *url in enumerator) { -// NSNumber *isDirectory = nil; -// NSError *error = nil; -// if (![url getResourceValue:&isDirectory forKey:NSURLIsDirectoryKey error:&error]) { -// break; -// } else if ([isDirectory boolValue]) { -// continue; -// } -// NSString *full = [url path]; -// NSRange r = [full rangeOfString:native]; -// if (r.location != 0) { -// break; -// } -// NSString *file = [full substringFromIndex:r.length + 1]; -// auto name = QString::fromUtf8([file cStringUsingEncoding:NSUTF8StringEncoding]); -// if (!skip.contains(name)) { -// result.push_back(std::move(name)); -// } -// if (result.size() == limit) { -// break; -// } -// } -// return result; -} - -bool RemoveLegacyFile(const QString &path) { - const auto native = QFile::encodeName(path); - return unlink(native.constData()) == 0; -} - -} // namespace details -} // namespace Storage diff --git a/Telegram/SourceFiles/storage/storage_clear_legacy_win.cpp b/Telegram/SourceFiles/storage/storage_clear_legacy_win.cpp deleted file mode 100644 index c93a93992..000000000 --- a/Telegram/SourceFiles/storage/storage_clear_legacy_win.cpp +++ /dev/null @@ -1,62 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#include "storage/storage_clear_legacy.h" - -#include "base/platform/win/base_windows_h.h" - -namespace Storage { -namespace details { - -std::vector CollectFiles( - const QString &base, - size_type limit, - const base::flat_set &skip) { - Expects(base.endsWith('/')); - Expects(limit > 0); - - const auto native = QDir::toNativeSeparators(base).toStdWString(); - const auto search = native + L'*'; - - auto data = WIN32_FIND_DATA{ 0 }; - const auto handle = FindFirstFileEx( - search.c_str(), - FindExInfoBasic, - &data, - FindExSearchNameMatch, - nullptr, - 0); - if (handle == INVALID_HANDLE_VALUE) { - return {}; - } - const auto guard = gsl::finally([&] { FindClose(handle); }); - - auto result = std::vector(); - do { - const auto full = native + data.cFileName; - if (data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { - continue; - } - const auto file = QString::fromWCharArray( - data.cFileName, - full.size() - native.size()); - auto name = QDir::fromNativeSeparators(file); - if (!skip.contains(name)) { - result.push_back(std::move(name)); - } - } while (result.size() != limit && FindNextFile(handle, &data)); - - return result; -} - -bool RemoveLegacyFile(const QString &path) { - const auto native = QDir::toNativeSeparators(path).toStdWString(); - return (::DeleteFile(native.c_str()) != 0); -} - -} // namespace details -} // namespace Storage diff --git a/Telegram/SourceFiles/storage/storage_databases.cpp b/Telegram/SourceFiles/storage/storage_databases.cpp deleted file mode 100644 index 83bd1cd9e..000000000 --- a/Telegram/SourceFiles/storage/storage_databases.cpp +++ /dev/null @@ -1,104 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#include "storage/storage_databases.h" - -#include "storage/cache/storage_cache_database.h" - -namespace Storage { - -DatabasePointer::DatabasePointer( - not_null owner, - const std::unique_ptr &value) -: _value(value.get()) -, _owner(owner) { -} - -DatabasePointer::DatabasePointer(DatabasePointer &&other) -: _value(base::take(other._value)) -, _owner(other._owner) { -} - -DatabasePointer &DatabasePointer::operator=(DatabasePointer &&other) { - if (this != &other) { - destroy(); - _owner = other._owner; - _value = base::take(other._value); - } - return *this; -} - -DatabasePointer::~DatabasePointer() { - destroy(); -} - -Cache::Database *DatabasePointer::get() const { - return _value; -} - -Cache::Database &DatabasePointer::operator*() const { - Expects(_value != nullptr); - - return *get(); -} - -Cache::Database *DatabasePointer::operator->() const { - Expects(_value != nullptr); - - return get(); -} - -DatabasePointer::operator bool() const { - return get() != nullptr; -} - -void DatabasePointer::destroy() { - if (const auto value = base::take(_value)) { - _owner->destroy(value); - } -} - -Databases::Kept::Kept(std::unique_ptr &&database) -: database(std::move(database)) { -} - -DatabasePointer Databases::get( - const QString &path, - const Cache::details::Settings &settings) { - if (const auto i = _map.find(path); i != end(_map)) { - auto &kept = i->second; - Assert(kept.destroying.alive()); - kept.destroying = nullptr; - kept.database->reconfigure(settings); - return DatabasePointer(this, kept.database); - } - const auto [i, ok] = _map.emplace( - path, - std::make_unique(path, settings)); - return DatabasePointer(this, i->second.database); -} - -void Databases::destroy(Cache::Database *database) { - for (auto &entry : _map) { - const auto &path = entry.first; // Need to capture it in lambda. - auto &kept = entry.second; - if (kept.database.get() == database) { - Assert(!kept.destroying.alive()); - database->close(); - database->waitForCleaner([ - =, - guard = kept.destroying.make_guard() - ]() mutable { - crl::on_main(std::move(guard), [=] { - _map.erase(path); - }); - }); - } - } -} - -} // namespace Storage diff --git a/Telegram/SourceFiles/storage/storage_databases.h b/Telegram/SourceFiles/storage/storage_databases.h deleted file mode 100644 index 9d4832f4f..000000000 --- a/Telegram/SourceFiles/storage/storage_databases.h +++ /dev/null @@ -1,71 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#pragma once - -#include "storage/cache/storage_cache_database.h" -#include "base/binary_guard.h" - -namespace Storage { -namespace Cache { -namespace details { -struct Settings; -} // namespace details -class Database; -} // namespace Cache - -class Databases; - -class DatabasePointer { -public: - DatabasePointer(const DatabasePointer &other) = delete; - DatabasePointer(DatabasePointer &&other); - DatabasePointer &operator=(const DatabasePointer &other) = delete; - DatabasePointer &operator=(DatabasePointer &&other); - ~DatabasePointer(); - - Cache::Database *get() const; - Cache::Database &operator*() const; - Cache::Database *operator->() const; - explicit operator bool() const; - -private: - friend class Databases; - - DatabasePointer( - not_null owner, - const std::unique_ptr &value); - void destroy(); - - Cache::Database *_value = nullptr; - not_null _owner; - -}; - -class Databases { -public: - DatabasePointer get( - const QString &path, - const Cache::details::Settings &settings); - -private: - friend class DatabasePointer; - - struct Kept { - Kept(std::unique_ptr &&database); - - std::unique_ptr database; - base::binary_guard destroying; - }; - - void destroy(Cache::Database *database); - - std::map _map; - -}; - -} // namespace Storage diff --git a/Telegram/SourceFiles/storage/storage_encrypted_file.cpp b/Telegram/SourceFiles/storage/storage_encrypted_file.cpp deleted file mode 100644 index c2c8d1339..000000000 --- a/Telegram/SourceFiles/storage/storage_encrypted_file.cpp +++ /dev/null @@ -1,348 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#include "storage/storage_encrypted_file.h" - -#include "base/openssl_help.h" - -namespace Storage { -namespace { - -constexpr auto kBlockSize = CtrState::kBlockSize; - -enum class Format : uint32 { - Format_0, -}; - -struct BasicHeader { - BasicHeader(); - - void setFormat(Format format) { - this->format = static_cast(format); - } - Format getFormat() const { - return static_cast(format); - } - - bytes::array salt = { { bytes::type() } }; - uint32 format : 8; - uint32 reserved1 : 24; - uint32 reserved2 = 0; - uint64 applicationVersion = 0; - bytes::array checksum = { { bytes::type() } }; -}; - -BasicHeader::BasicHeader() -: format(static_cast(Format::Format_0)) -, reserved1(0) { -} - -} // namespace - -File::Result File::open( - const QString &path, - Mode mode, - const EncryptionKey &key) { - close(); - - const auto info = QFileInfo(path); - const auto dir = info.absoluteDir(); - if (mode != Mode::Read && !dir.exists()) { - if (!QDir().mkpath(dir.absolutePath())) { - return Result::Failed; - } - } - - _data.setFileName(info.absoluteFilePath()); - const auto result = attemptOpen(mode, key); - if (result != Result::Success) { - close(); - } - return result; - - static_assert(sizeof(BasicHeader) == kSaltSize - + sizeof(uint64) * 2 - + openssl::kSha256Size, "Unexpected paddings in the header."); - static_assert( - (sizeof(BasicHeader) - kSaltSize) % kBlockSize == 0, - "Not way to encrypt the header."); -} - -File::Result File::attemptOpen(Mode mode, const EncryptionKey &key) { - switch (mode) { - case Mode::Read: return attemptOpenForRead(key); - case Mode::ReadAppend: return attemptOpenForReadAppend(key); - case Mode::Write: return attemptOpenForWrite(key); - } - Unexpected("Mode in Storage::File::attemptOpen."); -} - -File::Result File::attemptOpenForRead(const EncryptionKey &key) { - if (!_data.open(QIODevice::ReadOnly)) { - return Result::Failed; - } - return readHeader(key); -} - -File::Result File::attemptOpenForReadAppend(const EncryptionKey &key) { - if (!_lock.lock(_data, QIODevice::ReadWrite)) { - return Result::LockFailed; - } - const auto size = _data.size(); - if (!size) { - return writeHeader(key) ? Result::Success : Result::Failed; - } - return readHeader(key); -} - -File::Result File::attemptOpenForWrite(const EncryptionKey &key) { - if (!_lock.lock(_data, QIODevice::WriteOnly)) { - return Result::LockFailed; - } - return writeHeader(key) ? Result::Success : Result::Failed; -} - -bool File::writeHeader(const EncryptionKey &key) { - Expects(!_state.has_value()); - Expects(_data.pos() == 0); - - const auto magic = bytes::make_span("TDEF"); - if (!writePlain(magic.subspan(0, FileLock::kSkipBytes))) { - return false; - } - - auto header = BasicHeader(); - bytes::set_random(header.salt); - _state = key.prepareCtrState(header.salt); - - const auto headerBytes = bytes::object_as_span(&header); - const auto checkSize = headerBytes.size() - header.checksum.size(); - bytes::copy( - header.checksum, - openssl::Sha256( - key.data(), - headerBytes.subspan(0, checkSize))); - - if (writePlain(header.salt) != header.salt.size()) { - return false; - } else if (!write(headerBytes.subspan(header.salt.size()))) { - return false; - } - _dataSize = 0; - return true; -} - -File::Result File::readHeader(const EncryptionKey &key) { - Expects(!_state.has_value()); - Expects(_data.pos() == 0); - - if (!_data.seek(FileLock::kSkipBytes)) { - return Result::Failed; - } - auto header = BasicHeader(); - const auto headerBytes = bytes::object_as_span(&header); - if (readPlain(headerBytes) != headerBytes.size()) { - return Result::Failed; - } - _state = key.prepareCtrState(header.salt); - decrypt(headerBytes.subspan(header.salt.size())); - - const auto checkSize = headerBytes.size() - header.checksum.size(); - const auto checksum = openssl::Sha256( - key.data(), - headerBytes.subspan(0, checkSize)); - if (bytes::compare(header.checksum, checksum) != 0) { - return Result::WrongKey; - } else if (header.getFormat() != Format::Format_0) { - return Result::Failed; - } - _dataSize = _data.size() - - int64(sizeof(BasicHeader)) - - FileLock::kSkipBytes; - Assert(_dataSize >= 0); - if (const auto bad = (_dataSize % kBlockSize)) { - _dataSize -= bad; - } - return Result::Success; -} - -size_type File::readPlain(bytes::span bytes) { - return _data.read(reinterpret_cast(bytes.data()), bytes.size()); -} - -size_type File::writePlain(bytes::const_span bytes) { - return _data.write( - reinterpret_cast(bytes.data()), - bytes.size()); -} - -void File::decrypt(bytes::span bytes) { - Expects(_state.has_value()); - - _state->decrypt(bytes, _encryptionOffset); - _encryptionOffset += bytes.size(); -} - -void File::encrypt(bytes::span bytes) { - Expects(_state.has_value()); - - _state->encrypt(bytes, _encryptionOffset); - _encryptionOffset += bytes.size(); -} - -size_type File::read(bytes::span bytes) { - Expects(bytes.size() % kBlockSize == 0); - - auto count = readPlain(bytes); - if (const auto back = -(count % kBlockSize)) { - if (!_data.seek(_data.pos() + back)) { - return 0; - } - count += back; - } - if (count) { - decrypt(bytes.subspan(0, count)); - } - return count; -} - -bool File::write(bytes::span bytes) { - Expects(bytes.size() % kBlockSize == 0); - - if (!isOpen()) { - return false; - } - encrypt(bytes); - const auto count = writePlain(bytes); - if (count == bytes.size()) { - _dataSize = std::max(_dataSize, offset()); - } else { - decryptBack(bytes); - if (count > 0) { - _data.seek(_data.pos() - count); - } - return false; - } - return true; -} - -void File::decryptBack(bytes::span bytes) { - Expects(_encryptionOffset >= bytes.size()); - - _encryptionOffset -= bytes.size(); - decrypt(bytes); - _encryptionOffset -= bytes.size(); -} - -size_type File::readWithPadding(bytes::span bytes) { - const auto size = bytes.size(); - const auto part = size % kBlockSize; - const auto good = size - part; - if (good) { - const auto succeed = read(bytes.subspan(0, good)); - if (succeed != good) { - return succeed; - } - } - if (!part) { - return good; - } - auto storage = bytes::array(); - const auto padded = bytes::make_span(storage); - const auto succeed = read(padded); - if (!succeed) { - return good; - } - Assert(succeed == kBlockSize); - bytes::copy(bytes.subspan(good), padded.subspan(0, part)); - return size; -} - -bool File::writeWithPadding(bytes::span bytes) { - const auto size = bytes.size(); - const auto part = size % kBlockSize; - const auto good = size - part; - if (good && !write(bytes.subspan(0, good))) { - return false; - } - if (!part) { - return true; - } - auto storage = bytes::array(); - const auto padded = bytes::make_span(storage); - bytes::copy(padded, bytes.subspan(good)); - bytes::set_random(padded.subspan(part)); - if (write(padded)) { - return true; - } - if (good) { - decryptBack(bytes.subspan(0, good)); - _data.seek(_data.pos() - good); - } - return false; -} - -bool File::flush() { - return _data.flush(); -} - -void File::close() { - _lock.unlock(); - _data.close(); - _data.setFileName(QString()); - _dataSize = _encryptionOffset = 0; - _state = std::nullopt; -} - -bool File::isOpen() const { - return _data.isOpen(); -} - -int64 File::size() const { - return _dataSize; -} - -int64 File::offset() const { - const auto realOffset = kSaltSize + _encryptionOffset; - const auto skipOffset = sizeof(BasicHeader); - return (realOffset >= skipOffset) ? (realOffset - skipOffset) : 0; -} - -bool File::seek(int64 offset) { - const auto realOffset = sizeof(BasicHeader) + offset; - if (offset < 0 || offset > _dataSize) { - return false; - } else if (!_data.seek(FileLock::kSkipBytes + realOffset)) { - return false; - } - _encryptionOffset = realOffset - kSaltSize; - return true; -} - -bool File::Move(const QString &from, const QString &to) { - QFile source(from); - if (!source.exists()) { - return false; - } - QFile destination(to); - if (destination.exists()) { - { - FileLock locker; - if (!locker.lock(destination, QIODevice::WriteOnly)) { - return false; - } - } - destination.close(); - if (!destination.remove()) { - return false; - } - } - return source.rename(to); -} - - -} // namespace Storage diff --git a/Telegram/SourceFiles/storage/storage_encrypted_file.h b/Telegram/SourceFiles/storage/storage_encrypted_file.h deleted file mode 100644 index f4756b6f4..000000000 --- a/Telegram/SourceFiles/storage/storage_encrypted_file.h +++ /dev/null @@ -1,73 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#pragma once - -#include "storage/storage_file_lock.h" -#include "storage/storage_encryption.h" -#include "base/bytes.h" -#include "base/optional.h" - -namespace Storage { - -class File { -public: - enum class Mode { - Read, - ReadAppend, - Write, - }; - enum class Result { - Failed, - LockFailed, - WrongKey, - Success, - }; - Result open(const QString &path, Mode mode, const EncryptionKey &key); - - size_type read(bytes::span bytes); - bool write(bytes::span bytes); - - size_type readWithPadding(bytes::span bytes); - bool writeWithPadding(bytes::span bytes); - - bool flush(); - - bool isOpen() const; - int64 size() const; - int64 offset() const; - bool seek(int64 offset); - - void close(); - - static bool Move(const QString &from, const QString &to); - -private: - Result attemptOpen(Mode mode, const EncryptionKey &key); - Result attemptOpenForRead(const EncryptionKey &key); - Result attemptOpenForReadAppend(const EncryptionKey &key); - Result attemptOpenForWrite(const EncryptionKey &key); - - bool writeHeader(const EncryptionKey &key); - Result readHeader(const EncryptionKey &key); - - size_type readPlain(bytes::span bytes); - size_type writePlain(bytes::const_span bytes); - void decrypt(bytes::span bytes); - void encrypt(bytes::span bytes); - void decryptBack(bytes::span bytes); - - QFile _data; - FileLock _lock; - int64 _encryptionOffset = 0; - int64 _dataSize = 0; - - std::optional _state; - -}; - -} // namespace Storage diff --git a/Telegram/SourceFiles/storage/storage_encrypted_file_tests.cpp b/Telegram/SourceFiles/storage/storage_encrypted_file_tests.cpp deleted file mode 100644 index 4d522a93e..000000000 --- a/Telegram/SourceFiles/storage/storage_encrypted_file_tests.cpp +++ /dev/null @@ -1,247 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#include "catch.hpp" - -#include "storage/storage_encrypted_file.h" - -#include -#include - -#ifdef Q_OS_WIN -#include "platform/win/windows_dlls.h" -#endif // Q_OS_WIN - -#include - -#include -#ifdef Q_OS_MAC -#include -#elif defined Q_OS_LINUX // Q_OS_MAC -#include -#endif // Q_OS_MAC || Q_OS_LINUX - -extern int (*TestForkedMethod)(); - -const auto Key = Storage::EncryptionKey(bytes::make_vector( - bytes::make_span("\ -abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\ -abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\ -abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\ -abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\ -").subspan(0, Storage::EncryptionKey::kSize))); - -const auto Name = QString("test.file"); - -const auto Test1 = bytes::make_span("testbytetestbyte").subspan(0, 16); -const auto Test2 = bytes::make_span("bytetestbytetest").subspan(0, 16); - -struct ForkInit { - static int Method() { - Storage::File file; - const auto result = file.open( - Name, - Storage::File::Mode::ReadAppend, - Key); - if (result != Storage::File::Result::Success) { - return -1; - } - - auto data = bytes::vector(16); - const auto read = file.read(data); - if (read != data.size()) { - return -1; - } else if (data != bytes::make_vector(Test1)) { - return -1; - } - - if (!file.write(data) || !file.flush()) { - return -1; - } -#ifdef _DEBUG - while (true) { - std::this_thread::sleep_for(std::chrono::seconds(1)); - } -#else // _DEBUG - std::this_thread::sleep_for(std::chrono::seconds(1)); - return 0; -#endif // _DEBUG - } - ForkInit() { -#ifdef Q_OS_WIN - Platform::Dlls::start(); -#endif // Q_OS_WIN - - TestForkedMethod = &ForkInit::Method; - } - -}; - -ForkInit ForkInitializer; -QProcess ForkProcess; - -TEST_CASE("simple encrypted file", "[storage_encrypted_file]") { - SECTION("writing file") { - Storage::File file; - const auto result = file.open( - Name, - Storage::File::Mode::Write, - Key); - REQUIRE(result == Storage::File::Result::Success); - - auto data = bytes::make_vector(Test1); - const auto success = file.write(data); - REQUIRE(success); - } - SECTION("reading and writing file") { - Storage::File file; - const auto result = file.open( - Name, - Storage::File::Mode::ReadAppend, - Key); - REQUIRE(result == Storage::File::Result::Success); - - auto data = bytes::vector(Test1.size()); - const auto read = file.read(data); - REQUIRE(read == data.size()); - REQUIRE(data == bytes::make_vector(Test1)); - - data = bytes::make_vector(Test2); - const auto success = file.write(data); - REQUIRE(success); - } - SECTION("offset and seek") { - Storage::File file; - const auto result = file.open( - Name, - Storage::File::Mode::ReadAppend, - Key); - REQUIRE(result == Storage::File::Result::Success); - REQUIRE(file.offset() == 0); - REQUIRE(file.size() == Test1.size() + Test2.size()); - - const auto success1 = file.seek(Test1.size()); - REQUIRE(success1); - REQUIRE(file.offset() == Test1.size()); - - auto data = bytes::vector(Test2.size()); - const auto read = file.read(data); - REQUIRE(read == data.size()); - REQUIRE(data == bytes::make_vector(Test2)); - REQUIRE(file.offset() == Test1.size() + Test2.size()); - REQUIRE(file.size() == Test1.size() + Test2.size()); - - const auto success2 = file.seek(Test1.size()); - REQUIRE(success2); - REQUIRE(file.offset() == Test1.size()); - - data = bytes::make_vector(Test1); - const auto success3 = file.write(data) && file.write(data); - REQUIRE(success3); - - REQUIRE(file.offset() == 3 * Test1.size()); - REQUIRE(file.size() == 3 * Test1.size()); - } - SECTION("reading file") { - Storage::File file; - - const auto result = file.open( - Name, - Storage::File::Mode::Read, - Key); - REQUIRE(result == Storage::File::Result::Success); - - auto data = bytes::vector(32); - const auto read = file.read(data); - REQUIRE(read == data.size()); - REQUIRE(data == bytes::concatenate(Test1, Test1)); - } - SECTION("moving file") { - const auto result = Storage::File::Move(Name, "other.file"); - REQUIRE(result); - } -} - -TEST_CASE("two process encrypted file", "[storage_encrypted_file]") { - SECTION("writing file") { - Storage::File file; - const auto result = file.open( - Name, - Storage::File::Mode::Write, - Key); - REQUIRE(result == Storage::File::Result::Success); - - auto data = bytes::make_vector(Test1); - const auto success = file.write(data); - REQUIRE(success); - } - SECTION("access from subprocess") { - SECTION("start subprocess") { - const auto application = []() -> QString { -#ifdef Q_OS_WIN - return "tests_storage.exe"; -#else // Q_OS_WIN - constexpr auto kMaxPath = 1024; - char result[kMaxPath] = { 0 }; - uint32_t size = kMaxPath; -#ifdef Q_OS_MAC - if (_NSGetExecutablePath(result, &size) == 0) { - return result; - } -#else // Q_OS_MAC - auto count = readlink("/proc/self/exe", result, size); - if (count > 0) { - return result; - } -#endif // Q_OS_MAC - return "tests_storage"; -#endif // Q_OS_WIN - }(); - - ForkProcess.start(application + " --forked"); - const auto started = ForkProcess.waitForStarted(); - REQUIRE(started); - } - SECTION("read subprocess result") { - std::this_thread::sleep_for(std::chrono::milliseconds(500)); - - Storage::File file; - - const auto result = file.open( - Name, - Storage::File::Mode::Read, - Key); - REQUIRE(result == Storage::File::Result::Success); - - auto data = bytes::vector(32); - const auto read = file.read(data); - REQUIRE(read == data.size()); - REQUIRE(data == bytes::concatenate(Test1, Test1)); - } - SECTION("take subprocess result") { - REQUIRE(ForkProcess.state() == QProcess::Running); - - Storage::File file; - - const auto result = file.open( - Name, - Storage::File::Mode::ReadAppend, - Key); - REQUIRE(result == Storage::File::Result::Success); - - auto data = bytes::vector(32); - const auto read = file.read(data); - REQUIRE(read == data.size()); - REQUIRE(data == bytes::concatenate(Test1, Test1)); - - const auto finished = ForkProcess.waitForFinished(0); - REQUIRE(finished); - REQUIRE(ForkProcess.state() == QProcess::NotRunning); - } - } - -} diff --git a/Telegram/SourceFiles/storage/storage_encryption.cpp b/Telegram/SourceFiles/storage/storage_encryption.cpp deleted file mode 100644 index 61d0db392..000000000 --- a/Telegram/SourceFiles/storage/storage_encryption.cpp +++ /dev/null @@ -1,109 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#include "storage/storage_encryption.h" - -#include "base/openssl_help.h" - -namespace Storage { - -CtrState::CtrState(bytes::const_span key, bytes::const_span iv) { - Expects(key.size() == _key.size()); - Expects(iv.size() == _iv.size()); - - bytes::copy(_key, key); - bytes::copy(_iv, iv); -} - -template -void CtrState::process(bytes::span data, int64 offset, Method method) { - Expects((data.size() % kBlockSize) == 0); - Expects((offset % kBlockSize) == 0); - - AES_KEY aes; - AES_set_encrypt_key( - reinterpret_cast(_key.data()), - _key.size() * CHAR_BIT, - &aes); - - unsigned char ecountBuf[kBlockSize] = { 0 }; - unsigned int offsetInBlock = 0; - const auto blockIndex = offset / kBlockSize; - auto iv = incrementedIv(blockIndex); - - CRYPTO_ctr128_encrypt( - reinterpret_cast(data.data()), - reinterpret_cast(data.data()), - data.size(), - &aes, - reinterpret_cast(iv.data()), - ecountBuf, - &offsetInBlock, - (block128_f)method); -} - -auto CtrState::incrementedIv(int64 blockIndex) --> bytes::array { - Expects(blockIndex >= 0); - - if (!blockIndex) { - return _iv; - } - auto result = _iv; - auto digits = kIvSize; - auto increment = uint64(blockIndex); - do { - --digits; - increment += static_cast(result[digits]); - result[digits] = static_cast(increment & 0xFFULL); - increment >>= 8; - } while (digits != 0 && increment != 0); - return result; -} - -void CtrState::encrypt(bytes::span data, int64 offset) { - return process(data, offset, AES_encrypt); -} - -void CtrState::decrypt(bytes::span data, int64 offset) { - return process(data, offset, AES_encrypt); -} - -EncryptionKey::EncryptionKey(bytes::vector &&data) -: _data(std::move(data)) { - Expects(_data.size() == kSize); -} - -bool EncryptionKey::empty() const { - return _data.empty(); -} - -EncryptionKey::operator bool() const { - return !empty(); -} - -const bytes::vector &EncryptionKey::data() const { - return _data; -} - -CtrState EncryptionKey::prepareCtrState(bytes::const_span salt) const { - Expects(salt.size() == kSaltSize); - - const auto data = bytes::make_span(_data); - const auto key = openssl::Sha256( - data.subspan(0, kSize / 2), - salt.subspan(0, kSaltSize / 2)); - const auto iv = openssl::Sha256( - data.subspan(kSize / 2), - salt.subspan(kSaltSize / 2)); - - return CtrState( - key, - bytes::make_span(iv).subspan(0, CtrState::kIvSize)); -} - -} // namespace Storage diff --git a/Telegram/SourceFiles/storage/storage_encryption.h b/Telegram/SourceFiles/storage/storage_encryption.h deleted file mode 100644 index f366271ce..000000000 --- a/Telegram/SourceFiles/storage/storage_encryption.h +++ /dev/null @@ -1,58 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#pragma once - -#include "base/bytes.h" - -namespace Storage { - -constexpr auto kSaltSize = size_type(64); - -class CtrState { -public: - static constexpr auto kBlockSize = size_type(16); - static constexpr auto kKeySize = size_type(32); - static constexpr auto kIvSize = kBlockSize; - - CtrState(bytes::const_span key, bytes::const_span iv); - - void encrypt(bytes::span data, int64 offset); - void decrypt(bytes::span data, int64 offset); - -private: - template - void process(bytes::span data, int64 offset, Method method); - - bytes::array incrementedIv(int64 blockIndex); - - static constexpr auto EcountSize = kBlockSize; - - bytes::array _key; - bytes::array _iv; - -}; - -class EncryptionKey { -public: - static constexpr auto kSize = size_type(256); - - EncryptionKey() = default; - explicit EncryptionKey(bytes::vector &&data); - - bool empty() const; - explicit operator bool() const; - - const bytes::vector &data() const; - CtrState prepareCtrState(bytes::const_span salt) const; - -private: - bytes::vector _data; - -}; - -} // namespace Storage diff --git a/Telegram/SourceFiles/storage/storage_pch.cpp b/Telegram/SourceFiles/storage/storage_pch.cpp deleted file mode 100644 index 484defbf1..000000000 --- a/Telegram/SourceFiles/storage/storage_pch.cpp +++ /dev/null @@ -1,10 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ -#include "storage/storage_pch.h" - -// Precompiled header helper. diff --git a/Telegram/SourceFiles/storage/storage_pch.h b/Telegram/SourceFiles/storage/storage_pch.h deleted file mode 100644 index 986858f6a..000000000 --- a/Telegram/SourceFiles/storage/storage_pch.h +++ /dev/null @@ -1,30 +0,0 @@ -/* -This file is part of Telegram Desktop, -the official desktop application for the Telegram messaging service. - -For license and copyright information please follow this link: -https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL -*/ - -#include -#include -#include -#include -#include - -#include -#include - -#include -#include -#include -#include - -#include - -#include "base/flat_map.h" -#include "base/flat_set.h" -#include "base/optional.h" -#include "base/openssl_help.h" - -#include "logs.h"