diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_database.cpp b/Telegram/SourceFiles/storage/cache/storage_cache_database.cpp index 33324da10..75c98ccf3 100644 --- a/Telegram/SourceFiles/storage/cache/storage_cache_database.cpp +++ b/Telegram/SourceFiles/storage/cache/storage_cache_database.cpp @@ -16,16 +16,16 @@ Database::Database(const QString &path, const Settings &settings) : _wrapped(path, settings) { } -void Database::open(EncryptionKey key, FnMut done) { +void Database::open(EncryptionKey &&key, FnMut &&done) { _wrapped.with([ - key, + key = std::move(key), done = std::move(done) ](Implementation &unwrapped) mutable { - unwrapped.open(key, std::move(done)); + unwrapped.open(std::move(key), std::move(done)); }); } -void Database::close(FnMut done) { +void Database::close(FnMut &&done) { _wrapped.with([ done = std::move(done) ](Implementation &unwrapped) mutable { @@ -35,27 +35,23 @@ void Database::close(FnMut done) { void Database::put( const Key &key, - QByteArray value, - FnMut done) { - _wrapped.with([ - key, - value = std::move(value), - done = std::move(done) - ](Implementation &unwrapped) mutable { - unwrapped.put(key, std::move(value), std::move(done)); - }); + QByteArray &&value, + FnMut &&done) { + return put(key, TaggedValue(std::move(value), 0), std::move(done)); } -void Database::get(const Key &key, FnMut done) { - _wrapped.with([ - key, - done = std::move(done) - ](Implementation &unwrapped) mutable { - unwrapped.get(key, std::move(done)); - }); +void Database::get(const Key &key, FnMut &&done) { + if (done) { + auto untag = [done = std::move(done)](TaggedValue &&value) mutable { + done(std::move(value.bytes)); + }; + getWithTag(key, std::move(untag)); + } else { + getWithTag(key, nullptr); + } } -void Database::remove(const Key &key, FnMut done) { +void Database::remove(const Key &key, FnMut &&done) { _wrapped.with([ key, done = std::move(done) @@ -66,21 +62,18 @@ void Database::remove(const Key &key, FnMut done) { void Database::putIfEmpty( const Key &key, - QByteArray value, - FnMut done) { - _wrapped.with([ + QByteArray &&value, + FnMut &&done) { + return putIfEmpty( key, - value = std::move(value), - done = std::move(done) - ](Implementation &unwrapped) mutable { - unwrapped.putIfEmpty(key, std::move(value), std::move(done)); - }); + TaggedValue(std::move(value), 0), + std::move(done)); } void Database::copyIfEmpty( const Key &from, const Key &to, - FnMut done) { + FnMut &&done) { _wrapped.with([ from, to, @@ -93,7 +86,7 @@ void Database::copyIfEmpty( void Database::moveIfEmpty( const Key &from, const Key &to, - FnMut done) { + FnMut &&done) { _wrapped.with([ from, to, @@ -103,7 +96,52 @@ void Database::moveIfEmpty( }); } -void Database::clear(FnMut done) { +void Database::put( + const Key &key, + TaggedValue &&value, + FnMut &&done) { + _wrapped.with([ + key, + value = std::move(value), + done = std::move(done) + ](Implementation &unwrapped) mutable { + unwrapped.put(key, std::move(value), std::move(done)); + }); +} + +void Database::putIfEmpty( + const Key &key, + TaggedValue &&value, + FnMut &&done) { + _wrapped.with([ + key, + value = std::move(value), + done = std::move(done) + ](Implementation &unwrapped) mutable { + unwrapped.putIfEmpty(key, std::move(value), std::move(done)); + }); +} + +void Database::getWithTag( + const Key &key, + FnMut &&done) { + _wrapped.with([ + key, + done = std::move(done) + ](Implementation &unwrapped) mutable { + unwrapped.get(key, std::move(done)); + }); +} + +void Database::stats(FnMut &&done) { + _wrapped.with([ + done = std::move(done) + ](Implementation &unwrapped) mutable { + unwrapped.stats(std::move(done)); + }); +} + +void Database::clear(FnMut &&done) { _wrapped.with([ done = std::move(done) ](Implementation &unwrapped) mutable { diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_database.h b/Telegram/SourceFiles/storage/cache/storage_cache_database.h index 62a5258cd..46ff32c18 100644 --- a/Telegram/SourceFiles/storage/cache/storage_cache_database.h +++ b/Telegram/SourceFiles/storage/cache/storage_cache_database.h @@ -25,30 +25,44 @@ public: using Settings = details::Settings; Database(const QString &path, const Settings &settings); - void open(EncryptionKey key, FnMut done = nullptr); - void close(FnMut done = nullptr); + void open(EncryptionKey &&key, FnMut &&done = nullptr); + void close(FnMut &&done = nullptr); void put( const Key &key, - QByteArray value, - FnMut done = nullptr); - void get(const Key &key, FnMut done); - void remove(const Key &key, FnMut done = nullptr); + QByteArray &&value, + FnMut &&done = nullptr); + void get(const Key &key, FnMut &&done); + void remove(const Key &key, FnMut &&done = nullptr); void putIfEmpty( const Key &key, - QByteArray value, - FnMut done = nullptr); + QByteArray &&value, + FnMut &&done = nullptr); void copyIfEmpty( const Key &from, const Key &to, - FnMut done = nullptr); + FnMut &&done = nullptr); void moveIfEmpty( const Key &from, const Key &to, - FnMut done = nullptr); + FnMut &&done = nullptr); - void clear(FnMut done = nullptr); + using TaggedValue = details::TaggedValue; + void put( + const Key &key, + TaggedValue &&value, + FnMut &&done = nullptr); + void putIfEmpty( + const Key &key, + TaggedValue &&value, + FnMut &&done = nullptr); + void getWithTag(const Key &key, FnMut &&done); + + using Stats = details::Stats; + void stats(FnMut &&done); + + void clear(FnMut &&done = nullptr); ~Database(); diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_database_object.cpp b/Telegram/SourceFiles/storage/cache/storage_cache_database_object.cpp index 1a218f298..4d3b0024f 100644 --- a/Telegram/SourceFiles/storage/cache/storage_cache_database_object.cpp +++ b/Telegram/SourceFiles/storage/cache/storage_cache_database_object.cpp @@ -105,17 +105,17 @@ Error DatabaseObject::ioError(const QString &path) const { return { Error::Type::IO, path }; } -void DatabaseObject::open(EncryptionKey key, FnMut done) { +void DatabaseObject::open(EncryptionKey &&key, FnMut &&done) { close(nullptr); - const auto error = openSomeBinlog(key); + const auto error = openSomeBinlog(std::move(key)); if (error.type != Error::Type::None) { close(nullptr); } invokeCallback(done, error); } -Error DatabaseObject::openSomeBinlog(EncryptionKey &key) { +Error DatabaseObject::openSomeBinlog(EncryptionKey &&key) { const auto version = readVersion(); const auto result = openBinlog(version, File::Mode::ReadAppend, key); switch (result) { @@ -500,7 +500,7 @@ bool DatabaseObject::processRecordMultiAccess( void DatabaseObject::setMapEntry(const Key &key, Entry &&entry) { auto &already = _map[key]; - _totalSize += entry.size - already.size; + updateStats(already, entry); if (already.size != 0) { _binlogExcessLength += _settings.trackEstimatedTime ? sizeof(StoreWithTime) @@ -522,10 +522,32 @@ void DatabaseObject::setMapEntry(const Key &key, Entry &&entry) { already = std::move(entry); } +void DatabaseObject::updateStats(const Entry &was, const Entry &now) { + _totalSize += now.size - was.size; + if (now.tag == was.tag) { + if (now.tag) { + auto &summary = _taggedStats[now.tag]; + summary.count += (now.size ? 1 : 0) - (was.size ? 1 : 0); + summary.totalSize += now.size - was.size; + } + } else { + if (now.tag) { + auto &summary = _taggedStats[now.tag]; + summary.count += (now.size ? 1 : 0); + summary.totalSize += now.size; + } + if (was.tag) { + auto &summary = _taggedStats[was.tag]; + summary.count -= (was.size ? 1 : 0); + summary.totalSize -= was.size; + } + } +} + void DatabaseObject::eraseMapEntry(const Map::const_iterator &i) { if (i != end(_map)) { const auto &entry = i->second; - _totalSize -= entry.size; + updateStats(entry, Entry()); if (_minimalEntryTime != 0 && entry.useTime == _minimalEntryTime) { Assert(_entriesWithMinimalTimeCount > 0); --_entriesWithMinimalTimeCount; @@ -604,7 +626,7 @@ void DatabaseObject::compactorFail() { QFile(compactReadyPath()).remove(); } -void DatabaseObject::close(FnMut done) { +void DatabaseObject::close(FnMut &&done) { if (_binlog.isOpen()) { writeBundles(); } @@ -619,15 +641,15 @@ void DatabaseObject::close(FnMut done) { void DatabaseObject::put( const Key &key, - QByteArray value, - FnMut done) { - if (value.isEmpty()) { + TaggedValue &&value, + FnMut &&done) { + if (value.bytes.isEmpty()) { remove(key, std::move(done)); return; } _removing.erase(key); - const auto checksum = CountChecksum(bytes::make_span(value)); + const auto checksum = CountChecksum(bytes::make_span(value.bytes)); const auto maybepath = writeKeyPlace(key, value, checksum); if (!maybepath) { invokeCallback(done, ioError(binlogPath())); @@ -653,7 +675,8 @@ void DatabaseObject::put( break; case File::Result::Success: { - const auto success = data.writeWithPadding(bytes::make_span(value)); + const auto success = data.writeWithPadding( + bytes::make_span(value.bytes)); if (!success) { data.close(); remove(key, nullptr); @@ -673,11 +696,12 @@ template base::optional DatabaseObject::writeKeyPlaceGeneric( StoreRecord &&record, const Key &key, - const QByteArray &value, + const TaggedValue &value, uint32 checksum) { - Expects(value.size() <= _settings.maxDataSize); + Expects(value.bytes.size() <= _settings.maxDataSize); - const auto size = size_type(value.size()); + const auto size = size_type(value.bytes.size()); + record.tag = value.tag; record.key = key; record.size = ReadTo(size); record.checksum = checksum; @@ -686,7 +710,7 @@ base::optional DatabaseObject::writeKeyPlaceGeneric( if (already.tag == record.tag && already.size == size && already.checksum == checksum - && readValueData(already.place, size) == value) { + && readValueData(already.place, size) == value.bytes) { return QString(); } record.place = already.place; @@ -713,7 +737,7 @@ base::optional DatabaseObject::writeKeyPlaceGeneric( base::optional DatabaseObject::writeKeyPlace( const Key &key, - const QByteArray &data, + const TaggedValue &data, uint32 checksum) { if (!_settings.trackEstimatedTime) { return writeKeyPlaceGeneric(Store(), key, data, checksum); @@ -786,27 +810,25 @@ Error DatabaseObject::writeExistingPlace( return writeExistingPlaceGeneric(std::move(record), key, entry); } -void DatabaseObject::get(const Key &key, FnMut done) { - if (_removing.find(key) != end(_removing)) { - invokeCallback(done, QByteArray()); - return; - } +void DatabaseObject::get( + const Key &key, + FnMut &&done) { const auto i = _map.find(key); if (i == _map.end()) { - invokeCallback(done, QByteArray()); + invokeCallback(done, TaggedValue()); return; } const auto &entry = i->second; - auto result = readValueData(entry.place, entry.size); - if (result.isEmpty()) { + auto bytes = readValueData(entry.place, entry.size); + if (bytes.isEmpty()) { remove(key, nullptr); - invokeCallback(done, QByteArray()); - } else if (CountChecksum(bytes::make_span(result)) != entry.checksum) { + invokeCallback(done, TaggedValue()); + } else if (CountChecksum(bytes::make_span(bytes)) != entry.checksum) { remove(key, nullptr); - invokeCallback(done, QByteArray()); + invokeCallback(done, TaggedValue()); } else { - invokeCallback(done, std::move(result)); + invokeCallback(done, TaggedValue(std::move(bytes), entry.tag)); recordEntryAccess(key); } } @@ -840,7 +862,7 @@ void DatabaseObject::recordEntryAccess(const Key &key) { optimize(); } -void DatabaseObject::remove(const Key &key, FnMut done) { +void DatabaseObject::remove(const Key &key, FnMut &&done) { const auto i = _map.find(key); if (i != _map.end()) { _removing.emplace(key); @@ -860,8 +882,8 @@ void DatabaseObject::remove(const Key &key, FnMut done) { void DatabaseObject::putIfEmpty( const Key &key, - QByteArray value, - FnMut done) { + TaggedValue &&value, + FnMut &&done) { if (_map.find(key) != end(_map)) { invokeCallback(done, Error::NoError()); return; @@ -872,20 +894,20 @@ void DatabaseObject::putIfEmpty( void DatabaseObject::copyIfEmpty( const Key &from, const Key &to, - FnMut done) { + FnMut &&done) { if (_map.find(to) != end(_map)) { invokeCallback(done, Error::NoError()); return; } - get(from, [&](QByteArray value) { - put(to, value, std::move(done)); + get(from, [&](TaggedValue &&value) { + put(to, std::move(value), std::move(done)); }); } void DatabaseObject::moveIfEmpty( const Key &from, const Key &to, - FnMut done) { + FnMut &&done) { if (_map.find(to) != end(_map)) { invokeCallback(done, Error::NoError()); return; @@ -908,6 +930,19 @@ void DatabaseObject::moveIfEmpty( invokeCallback(done, writeExistingPlace(to, entry)); } +void DatabaseObject::stats(FnMut &&done) { + auto result = _taggedStats; + auto zero = TaggedSummary(); + zero.count = _map.size(); + zero.totalSize = _totalSize; + for (const auto &summary : result) { + zero.count -= summary.second.count; + zero.totalSize -= summary.second.totalSize; + } + result[0] = zero; + invokeCallback(done, std::move(result)); +} + void DatabaseObject::writeBundlesLazy() { if (!_writeBundlesTimer.isActive()) { _writeBundlesTimer.callOnce(_settings.writeBundleDelay); @@ -1043,7 +1078,7 @@ void DatabaseObject::checkCompactor() { _compactor.excessLength = _binlogExcessLength; } -void DatabaseObject::clear(FnMut done) { +void DatabaseObject::clear(FnMut &&done) { Expects(_key.empty()); const auto version = findAvailableVersion(); @@ -1052,7 +1087,7 @@ void DatabaseObject::clear(FnMut done) { writeVersion(version) ? Error::NoError() : ioError(versionPath())); } -auto DatabaseObject::getManyRaw(const std::vector keys) const +auto DatabaseObject::getManyRaw(const std::vector &keys) const -> std::vector { auto result = std::vector(); result.reserve(keys.size()); diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_database_object.h b/Telegram/SourceFiles/storage/cache/storage_cache_database_object.h index 0bfe9dbe2..f5eee3116 100644 --- a/Telegram/SourceFiles/storage/cache/storage_cache_database_object.h +++ b/Telegram/SourceFiles/storage/cache/storage_cache_database_object.h @@ -30,27 +30,32 @@ public: const QString &path, const Settings &settings); - void open(EncryptionKey key, FnMut done); - void close(FnMut done); + void open(EncryptionKey &&key, FnMut &&done); + void close(FnMut &&done); - void put(const Key &key, QByteArray value, FnMut done); - void get(const Key &key, FnMut done); - void remove(const Key &key, FnMut done); + void put( + const Key &key, + TaggedValue &&value, + FnMut &&done); + void get(const Key &key, FnMut &&done); + void remove(const Key &key, FnMut &&done); void putIfEmpty( const Key &key, - QByteArray value, - FnMut done); + TaggedValue &&value, + FnMut &&done); void copyIfEmpty( const Key &from, const Key &to, - FnMut done); + FnMut &&done); void moveIfEmpty( const Key &from, const Key &to, - FnMut done); + FnMut &&done); - void clear(FnMut done); + void stats(FnMut &&done); + + void clear(FnMut &&done); static QString BinlogFilename(); static QString CompactReadyFilename(); @@ -74,7 +79,7 @@ public: uint8 tag = 0; }; using Raw = std::pair; - std::vector getManyRaw(const std::vector keys) const; + std::vector getManyRaw(const std::vector &keys) const; ~DatabaseObject(); @@ -101,7 +106,7 @@ private: QString binlogPath() const; QString compactReadyPath(Version version) const; QString compactReadyPath() const; - Error openSomeBinlog(EncryptionKey &key); + Error openSomeBinlog(EncryptionKey &&key); Error openNewBinlog(EncryptionKey &key); File::Result openBinlog( Version version, @@ -149,6 +154,7 @@ private: void collectSizePrune( base::flat_set &stale, int64 &staleTotalSize); + void updateStats(const Entry &was, const Entry &now); void setMapEntry(const Key &key, Entry &&entry); void eraseMapEntry(const Map::const_iterator &i); @@ -167,11 +173,11 @@ private: base::optional writeKeyPlaceGeneric( StoreRecord &&record, const Key &key, - const QByteArray &value, + const TaggedValue &value, uint32 checksum); base::optional writeKeyPlace( const Key &key, - const QByteArray &value, + const TaggedValue &value, uint32 checksum); template Error writeExistingPlaceGeneric( @@ -208,6 +214,8 @@ private: uint64 _minimalEntryTime = 0; size_type _entriesWithMinimalTimeCount = 0; + Stats _taggedStats; + base::ConcurrentTimer _writeBundlesTimer; base::ConcurrentTimer _pruneTimer; diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_database_tests.cpp b/Telegram/SourceFiles/storage/cache/storage_cache_database_tests.cpp index 2cad84967..736e8bb68 100644 --- a/Telegram/SourceFiles/storage/cache/storage_cache_database_tests.cpp +++ b/Telegram/SourceFiles/storage/cache/storage_cache_database_tests.cpp @@ -59,8 +59,14 @@ QString GetBinlogPath() { return name + '/' + QString::number(version) + "/binlog"; } -const auto TestValue1 = QByteArray("testbytetestbyt"); -const auto TestValue2 = QByteArray("bytetestbytetestb"); +const auto TestValue1 = [] { + static auto result = QByteArray("testbytetestbyt"); + return result; +}; +const auto TestValue2 = [] { + static auto result = QByteArray("bytetestbytetestb"); + return result; +}; crl::semaphore Semaphore; @@ -77,7 +83,7 @@ const auto GetValue = [](QByteArray value) { }; Error Open(Database &db, const Storage::EncryptionKey &key) { - db.open(key, GetResult); + db.open(base::duplicate(key), GetResult); Semaphore.acquire(); return Result; } @@ -99,8 +105,8 @@ QByteArray Get(Database &db, const Key &key) { return Value; } -Error Put(Database &db, const Key &key, const QByteArray &value) { - db.put(key, value, GetResult); +Error Put(Database &db, const Key &key, QByteArray &&value) { + db.put(key, std::move(value), GetResult); Semaphore.acquire(); return Result; } @@ -141,15 +147,15 @@ TEST_CASE("compacting db", "[storage_cache_database]") { for (auto i = from; i != till; ++i) { auto value = base; value[0] = char('A') + i; - const auto result = Put(db, Key{ i, i + 1 }, value); + const auto result = Put(db, Key{ i, i + 1 }, std::move(value)); REQUIRE(result.type == Error::Type::None); } }; const auto put = [&](Database &db, uint32 from, uint32 till) { - write(db, from, till, TestValue1); + write(db, from, till, TestValue1()); }; const auto reput = [&](Database &db, uint32 from, uint32 till) { - write(db, from, till, TestValue2); + write(db, from, till, TestValue2()); }; const auto remove = [](Database &db, uint32 from, uint32 till) { for (auto i = from; i != till; ++i) { @@ -201,11 +207,11 @@ TEST_CASE("compacting db", "[storage_cache_database]") { const auto fullcheck = [&] { check(db, 0, 15, {}); - check(db, 15, 20, TestValue1); - check(db, 20, 30, TestValue2); + check(db, 15, 20, TestValue1()); + check(db, 20, 30, TestValue2()); check(db, 30, 35, {}); - check(db, 35, 37, TestValue2); - check(db, 37, 45, TestValue1); + check(db, 35, 37, TestValue2()); + check(db, 37, 45, TestValue1()); }; fullcheck(); Close(db); @@ -241,11 +247,11 @@ TEST_CASE("compacting db", "[storage_cache_database]") { const auto fullcheck = [&] { check(db, 0, 15, {}); - check(db, 15, 20, TestValue1); - check(db, 20, 30, TestValue2); + check(db, 15, 20, TestValue1()); + check(db, 20, 30, TestValue2()); check(db, 30, 35, {}); - check(db, 35, 37, TestValue2); - check(db, 37, 45, TestValue1); + check(db, 35, 37, TestValue2()); + check(db, 37, 45, TestValue1()); }; fullcheck(); Close(db); @@ -285,9 +291,9 @@ TEST_CASE("compacting db", "[storage_cache_database]") { REQUIRE(after < size2); const auto fullcheck = [&] { check(db, 0, 15, {}); - check(db, 15, 20, TestValue1); + check(db, 15, 20, TestValue1()); check(db, 20, 35, {}); - check(db, 35, 45, TestValue2); + check(db, 35, 45, TestValue2()); }; fullcheck(); Close(db); @@ -324,7 +330,7 @@ TEST_CASE("compacting db", "[storage_cache_database]") { AdvanceTime(2); REQUIRE(QFile(path).size() < size); const auto fullcheck = [&] { - check(db, 15, 30, TestValue2); + check(db, 15, 30, TestValue2()); }; fullcheck(); Close(db); @@ -344,16 +350,16 @@ TEST_CASE("encrypted cache db", "[storage_cache_database]") { REQUIRE(Clear(db).type == Error::Type::None); REQUIRE(Open(db, key).type == Error::Type::None); - REQUIRE(Put(db, Key{ 0, 1 }, TestValue1).type == Error::Type::None); + REQUIRE(Put(db, Key{ 0, 1 }, TestValue1()).type == Error::Type::None); Close(db); } SECTION("reading and writing db") { Database db(name, Settings); REQUIRE(Open(db, key).type == Error::Type::None); - REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1)); - REQUIRE(Put(db, Key{ 1, 0 }, TestValue2).type == Error::Type::None); - REQUIRE((Get(db, Key{ 1, 0 }) == TestValue2)); + REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1())); + REQUIRE(Put(db, Key{ 1, 0 }, TestValue2()).type == Error::Type::None); + REQUIRE((Get(db, Key{ 1, 0 }) == TestValue2())); REQUIRE(Get(db, Key{ 1, 1 }).isEmpty()); Close(db); } @@ -361,8 +367,8 @@ TEST_CASE("encrypted cache db", "[storage_cache_database]") { Database db(name, Settings); REQUIRE(Open(db, key).type == Error::Type::None); - REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1)); - REQUIRE((Get(db, Key{ 1, 0 }) == TestValue2)); + REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1())); + REQUIRE((Get(db, Key{ 1, 0 }) == TestValue2())); Close(db); } SECTION("overwriting values") { @@ -370,13 +376,13 @@ TEST_CASE("encrypted cache db", "[storage_cache_database]") { REQUIRE(Open(db, key).type == Error::Type::None); const auto path = GetBinlogPath(); - REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1)); + REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1())); const auto size = QFile(path).size(); - REQUIRE(Put(db, Key{ 0, 1 }, TestValue2).type == Error::Type::None); + REQUIRE(Put(db, Key{ 0, 1 }, TestValue2()).type == Error::Type::None); const auto next = QFile(path).size(); REQUIRE(next > size); - REQUIRE((Get(db, Key{ 0, 1 }) == TestValue2)); - REQUIRE(Put(db, Key{ 0, 1 }, TestValue2).type == Error::Type::None); + REQUIRE((Get(db, Key{ 0, 1 }) == TestValue2())); + REQUIRE(Put(db, Key{ 0, 1 }, TestValue2()).type == Error::Type::None); const auto same = QFile(path).size(); REQUIRE(same == next); Close(db); @@ -393,16 +399,16 @@ TEST_CASE("encrypted cache db", "[storage_cache_database]") { REQUIRE(Clear(db).type == Error::Type::None); REQUIRE(Open(db, key).type == Error::Type::None); for (auto i = 0U; i != count; ++i) { - auto value = TestValue1; + auto value = TestValue1(); value[0] = char('A') + i; - const auto result = Put(db, Key{ i, i * 2 }, value); + const auto result = Put(db, Key{ i, i * 2 }, std::move(value)); REQUIRE(result.type == Error::Type::None); } Close(db); REQUIRE(Open(db, key).type == Error::Type::None); for (auto i = 0U; i != count; ++i) { - auto value = TestValue1; + auto value = TestValue1(); value[0] = char('A') + i; REQUIRE((Get(db, Key{ i, i * 2 }) == value)); } @@ -419,11 +425,11 @@ TEST_CASE("cache db remove", "[storage_cache_database]") { REQUIRE(Clear(db).type == Error::Type::None); REQUIRE(Open(db, key).type == Error::Type::None); - REQUIRE(Put(db, Key{ 0, 1 }, TestValue1).type == Error::Type::None); - REQUIRE(Put(db, Key{ 1, 0 }, TestValue2).type == Error::Type::None); + REQUIRE(Put(db, Key{ 0, 1 }, TestValue1()).type == Error::Type::None); + REQUIRE(Put(db, Key{ 1, 0 }, TestValue2()).type == Error::Type::None); Remove(db, Key{ 0, 1 }); REQUIRE(Get(db, Key{ 0, 1 }).isEmpty()); - REQUIRE((Get(db, Key{ 1, 0 }) == TestValue2)); + REQUIRE((Get(db, Key{ 1, 0 }) == TestValue2())); Close(db); } SECTION("db remove deletes value permanently") { @@ -431,7 +437,7 @@ TEST_CASE("cache db remove", "[storage_cache_database]") { REQUIRE(Open(db, key).type == Error::Type::None); REQUIRE(Get(db, Key{ 0, 1 }).isEmpty()); - REQUIRE((Get(db, Key{ 1, 0 }) == TestValue2)); + REQUIRE((Get(db, Key{ 1, 0 }) == TestValue2())); Close(db); } } @@ -448,9 +454,9 @@ TEST_CASE("cache db bundled actions", "[storage_cache_database]") { REQUIRE(Clear(db).type == Error::Type::None); REQUIRE(Open(db, key).type == Error::Type::None); const auto path = GetBinlogPath(); - REQUIRE(Put(db, Key{ 0, 1 }, TestValue1).type == Error::Type::None); + REQUIRE(Put(db, Key{ 0, 1 }, TestValue1()).type == Error::Type::None); const auto size = QFile(path).size(); - REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1)); + REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1())); REQUIRE(QFile(path).size() == size); AdvanceTime(2); Get(db, Key{ 0, 1 }); @@ -465,9 +471,9 @@ TEST_CASE("cache db bundled actions", "[storage_cache_database]") { REQUIRE(Clear(db).type == Error::Type::None); REQUIRE(Open(db, key).type == Error::Type::None); const auto path = GetBinlogPath(); - REQUIRE(Put(db, Key{ 0, 1 }, TestValue1).type == Error::Type::None); + REQUIRE(Put(db, Key{ 0, 1 }, TestValue1()).type == Error::Type::None); const auto size = QFile(path).size(); - REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1)); + REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1())); REQUIRE(QFile(path).size() == size); Close(db); REQUIRE(QFile(path).size() > size); @@ -478,7 +484,7 @@ TEST_CASE("cache db bundled actions", "[storage_cache_database]") { REQUIRE(Clear(db).type == Error::Type::None); REQUIRE(Open(db, key).type == Error::Type::None); const auto path = GetBinlogPath(); - REQUIRE(Put(db, Key{ 0, 1 }, TestValue1).type == Error::Type::None); + REQUIRE(Put(db, Key{ 0, 1 }, TestValue1()).type == Error::Type::None); const auto size = QFile(path).size(); Remove(db, Key{ 0, 1 }); REQUIRE(QFile(path).size() == size); @@ -492,7 +498,7 @@ TEST_CASE("cache db bundled actions", "[storage_cache_database]") { REQUIRE(Clear(db).type == Error::Type::None); REQUIRE(Open(db, key).type == Error::Type::None); const auto path = GetBinlogPath(); - REQUIRE(Put(db, Key{ 0, 1 }, TestValue1).type == Error::Type::None); + REQUIRE(Put(db, Key{ 0, 1 }, TestValue1()).type == Error::Type::None); const auto size = QFile(path).size(); Remove(db, Key{ 0, 1 }); REQUIRE(QFile(path).size() == size); @@ -513,21 +519,21 @@ TEST_CASE("cache db limits", "[storage_cache_database]") { Database db(name, settings); db.clear(nullptr); - db.open(key, nullptr); - db.put(Key{ 0, 1 }, TestValue1, nullptr); - db.put(Key{ 1, 0 }, TestValue2, nullptr); + db.open(base::duplicate(key), nullptr); + db.put(Key{ 0, 1 }, TestValue1(), nullptr); + db.put(Key{ 1, 0 }, TestValue2(), nullptr); AdvanceTime(2); db.get(Key{ 1, 0 }, nullptr); AdvanceTime(3); - db.put(Key{ 1, 1 }, TestValue1, nullptr); - db.put(Key{ 2, 0 }, TestValue2, nullptr); - db.put(Key{ 0, 2 }, TestValue1, nullptr); + db.put(Key{ 1, 1 }, TestValue1(), nullptr); + db.put(Key{ 2, 0 }, TestValue2(), nullptr); + db.put(Key{ 0, 2 }, TestValue1(), nullptr); AdvanceTime(2); REQUIRE(Get(db, Key{ 0, 1 }).isEmpty()); REQUIRE(Get(db, Key{ 1, 0 }).isEmpty()); - REQUIRE((Get(db, Key{ 1, 1 }) == TestValue1)); - REQUIRE((Get(db, Key{ 2, 0 }) == TestValue2)); - REQUIRE((Get(db, Key{ 0, 2 }) == TestValue1)); + REQUIRE((Get(db, Key{ 1, 1 }) == TestValue1())); + REQUIRE((Get(db, Key{ 2, 0 }) == TestValue2())); + REQUIRE((Get(db, Key{ 0, 2 }) == TestValue1())); Close(db); } SECTION("db size limit") { @@ -537,27 +543,27 @@ TEST_CASE("cache db limits", "[storage_cache_database]") { Database db(name, settings); db.clear(nullptr); - db.open(key, nullptr); - db.put(Key{ 0, 1 }, TestValue1, nullptr); + db.open(base::duplicate(key), nullptr); + db.put(Key{ 0, 1 }, TestValue1(), nullptr); AdvanceTime(2); - db.put(Key{ 1, 0 }, TestValue2, nullptr); + db.put(Key{ 1, 0 }, TestValue2(), nullptr); AdvanceTime(2); - db.put(Key{ 1, 1 }, TestValue1, nullptr); + db.put(Key{ 1, 1 }, TestValue1(), nullptr); db.get(Key{ 0, 1 }, nullptr); AdvanceTime(2); - db.put(Key{ 2, 0 }, TestValue2, nullptr); + db.put(Key{ 2, 0 }, TestValue2(), nullptr); // Removing { 1, 0 } will be scheduled. - REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1)); - REQUIRE((Get(db, Key{ 1, 1 }) == TestValue1)); - REQUIRE((Get(db, Key{ 2, 0 }) == TestValue2)); + REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1())); + REQUIRE((Get(db, Key{ 1, 1 }) == TestValue1())); + REQUIRE((Get(db, Key{ 2, 0 }) == TestValue2())); AdvanceTime(2); // Removing { 1, 0 } performed. REQUIRE(Get(db, Key{ 1, 0 }).isEmpty()); - REQUIRE((Get(db, Key{ 1, 1 }) == TestValue1)); - db.put(Key{ 0, 2 }, TestValue1, nullptr); - REQUIRE(Put(db, Key{ 2, 2 }, TestValue2).type == Error::Type::None); + REQUIRE((Get(db, Key{ 1, 1 }) == TestValue1())); + db.put(Key{ 0, 2 }, TestValue1(), nullptr); + REQUIRE(Put(db, Key{ 2, 2 }, TestValue2()).type == Error::Type::None); // Removing { 0, 1 } and { 2, 0 } will be scheduled. AdvanceTime(2); @@ -565,9 +571,9 @@ TEST_CASE("cache db limits", "[storage_cache_database]") { // Removing { 0, 1 } and { 2, 0 } performed. REQUIRE(Get(db, Key{ 0, 1 }).isEmpty()); REQUIRE(Get(db, Key{ 2, 0 }).isEmpty()); - REQUIRE((Get(db, Key{ 1, 1 }) == TestValue1)); - REQUIRE((Get(db, Key{ 0, 2 }) == TestValue1)); - REQUIRE((Get(db, Key{ 2, 2 }) == TestValue2)); + REQUIRE((Get(db, Key{ 1, 1 }) == TestValue1())); + REQUIRE((Get(db, Key{ 0, 2 }) == TestValue1())); + REQUIRE((Get(db, Key{ 2, 2 }) == TestValue2())); Close(db); } SECTION("db time limit") { @@ -577,11 +583,11 @@ TEST_CASE("cache db limits", "[storage_cache_database]") { Database db(name, settings); db.clear(nullptr); - db.open(key, nullptr); - db.put(Key{ 0, 1 }, TestValue1, nullptr); - db.put(Key{ 1, 0 }, TestValue2, nullptr); - db.put(Key{ 1, 1 }, TestValue1, nullptr); - db.put(Key{ 2, 0 }, TestValue2, nullptr); + db.open(base::duplicate(key), nullptr); + db.put(Key{ 0, 1 }, TestValue1(), nullptr); + db.put(Key{ 1, 0 }, TestValue2(), nullptr); + db.put(Key{ 1, 1 }, TestValue1(), nullptr); + db.put(Key{ 2, 0 }, TestValue2(), nullptr); AdvanceTime(1); db.get(Key{ 1, 0 }, nullptr); db.get(Key{ 1, 1 }, nullptr); @@ -594,8 +600,8 @@ TEST_CASE("cache db limits", "[storage_cache_database]") { AdvanceTime(3); REQUIRE(Get(db, Key{ 2, 0 }).isEmpty()); REQUIRE(Get(db, Key{ 1, 1 }).isEmpty()); - REQUIRE((Get(db, Key{ 1, 0 }) == TestValue2)); - REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1)); + REQUIRE((Get(db, Key{ 1, 0 }) == TestValue2())); + REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1())); Close(db); } } @@ -623,7 +629,7 @@ TEST_CASE("large db", "[storage_cache_database]") { }; const auto kWriteRecords = 100 * 1024; for (auto i = 0; i != kWriteRecords; ++i) { - db.put(key(i), TestValue1, nullptr); + db.put(key(i), TestValue1(), nullptr); const auto j = i ? (rand() % i) : 0; if (i % 1024 == 1023) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_types.cpp b/Telegram/SourceFiles/storage/cache/storage_cache_types.cpp index f7fca7ee7..9aba5788f 100644 --- a/Telegram/SourceFiles/storage/cache/storage_cache_types.cpp +++ b/Telegram/SourceFiles/storage/cache/storage_cache_types.cpp @@ -13,6 +13,10 @@ namespace Storage { namespace Cache { namespace details { +TaggedValue::TaggedValue(QByteArray &&bytes, uint8 tag) +: bytes(std::move(bytes)), tag(tag) { +} + QString ComputeBasePath(const QString &original) { const auto result = QDir(original).absolutePath(); return result.endsWith('/') ? result : (result + '/'); diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_types.h b/Telegram/SourceFiles/storage/cache/storage_cache_types.h index 7828262da..04fe2e2d5 100644 --- a/Telegram/SourceFiles/storage/cache/storage_cache_types.h +++ b/Telegram/SourceFiles/storage/cache/storage_cache_types.h @@ -8,6 +8,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #pragma once #include "base/basic_types.h" +#include "base/flat_map.h" #include "base/optional.h" #include @@ -67,6 +68,20 @@ struct Settings { crl::time_type maxPruneCheckTimeout = 3600 * crl::time_type(1000); }; +struct TaggedValue { + TaggedValue() = default; + TaggedValue(QByteArray &&bytes, uint8 tag); + + QByteArray bytes; + uint8 tag = 0; +}; + +struct TaggedSummary { + size_type count = 0; + size_type totalSize = 0; +}; +using Stats = base::flat_map; + using Version = int32; QString ComputeBasePath(const QString &original); diff --git a/Telegram/SourceFiles/storage/file_download.cpp b/Telegram/SourceFiles/storage/file_download.cpp index 95828fb52..0bdfdb5dd 100644 --- a/Telegram/SourceFiles/storage/file_download.cpp +++ b/Telegram/SourceFiles/storage/file_download.cpp @@ -365,9 +365,9 @@ void FileLoader::loadLocal(const Storage::Cache::Key &key) { auto [first, second] = base::make_binary_guard(); _localLoading = std::move(first); auto done = [=, guard = std::move(second)]( - QByteArray value, - QImage image, - QByteArray format) mutable { + QByteArray &&value, + QImage &&image, + QByteArray &&format) mutable { crl::on_main([ =, value = std::move(value), @@ -385,7 +385,7 @@ void FileLoader::loadLocal(const Storage::Cache::Key &key) { }); }; Auth().data().cache().get(key, [=, callback = std::move(done)]( - QByteArray value) mutable { + QByteArray &&value) mutable { if (readImage) { crl::async([ value = std::move(value), @@ -394,13 +394,16 @@ void FileLoader::loadLocal(const Storage::Cache::Key &key) { auto format = QByteArray(); auto image = App::readImage(value, &format, false); if (!image.isNull()) { - done(value, image, format); + done( + std::move(value), + std::move(image), + std::move(format)); } else { - done(value, {}, {}); + done(std::move(value), {}, {}); } }); } else { - callback(value, {}, {}); + callback(std::move(value), {}, {}); } }); } @@ -954,7 +957,7 @@ bool mtpFileLoader::feedPart(int offset, bytes::const_span buffer) { || _locationType == UnknownFileLocation || _toCache == LoadToCacheAsWell) { if (const auto key = cacheKey()) { - Auth().data().cache().put(*key, _data); + Auth().data().cache().put(*key, base::duplicate(_data)); } } } @@ -1164,7 +1167,7 @@ void webFileLoader::onFinished(const QByteArray &data) { if (_localStatus == LocalStatus::NotFound) { if (const auto key = cacheKey()) { - Auth().data().cache().put(*key, _data); + Auth().data().cache().put(*key, base::duplicate(_data)); } } _downloader->taskFinished().notify(); diff --git a/Telegram/SourceFiles/storage/file_upload.cpp b/Telegram/SourceFiles/storage/file_upload.cpp index c999c8608..215b43dde 100644 --- a/Telegram/SourceFiles/storage/file_upload.cpp +++ b/Telegram/SourceFiles/storage/file_upload.cpp @@ -127,7 +127,9 @@ void Uploader::uploadMedia(const FullMsgId &msgId, const SendMediaReady &media) if (!media.data.isEmpty()) { document->setData(media.data); if (document->saveToCache()) { - Auth().data().cache().put(document->cacheKey(), media.data); + Auth().data().cache().put( + document->cacheKey(), + base::duplicate(media.data)); } } if (!media.file.isEmpty()) { @@ -154,7 +156,7 @@ void Uploader::upload( if (document->saveToCache()) { Auth().data().cache().put( document->cacheKey(), - file->content); + base::duplicate(file->content)); } } if (!file->filepath.isEmpty()) { diff --git a/Telegram/SourceFiles/ui/images.cpp b/Telegram/SourceFiles/ui/images.cpp index f207e9fcd..7e78ec1a2 100644 --- a/Telegram/SourceFiles/ui/images.cpp +++ b/Telegram/SourceFiles/ui/images.cpp @@ -1022,7 +1022,9 @@ void RemoteImage::setImageBytes( const auto location = this->location(); if (!location.isNull() && !bytes.isEmpty()) { - Auth().data().cache().putIfEmpty(Data::StorageCacheKey(location), bytes); + Auth().data().cache().putIfEmpty( + Data::StorageCacheKey(location), + base::duplicate(bytes)); } }