mirror of https://github.com/procxx/kepka.git
Removed old lib_storage sources.
This commit is contained in:
John Preston
parent
5d6fd32496
commit
2d7f6fc2e7
|
|
@ -1,111 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#include "storage/cache/storage_cache_binlog_reader.h"
|
||||
|
||||
namespace Storage {
|
||||
namespace Cache {
|
||||
namespace details {
|
||||
|
||||
BinlogWrapper::BinlogWrapper(
|
||||
File &binlog,
|
||||
const Settings &settings,
|
||||
int64 till)
|
||||
: _binlog(binlog)
|
||||
, _settings(settings)
|
||||
, _till(till ? till : _binlog.size())
|
||||
, _data(_settings.readBlockSize)
|
||||
, _full(_data) {
|
||||
}
|
||||
|
||||
bool BinlogWrapper::finished() const {
|
||||
return _finished;
|
||||
}
|
||||
|
||||
bool BinlogWrapper::failed() const {
|
||||
return _failed;
|
||||
}
|
||||
|
||||
std::optional<BasicHeader> BinlogWrapper::ReadHeader(
|
||||
File &binlog,
|
||||
const Settings &settings) {
|
||||
auto result = BasicHeader();
|
||||
if (binlog.offset() != 0) {
|
||||
return {};
|
||||
} else if (binlog.read(bytes::object_as_span(&result)) != sizeof(result)) {
|
||||
return {};
|
||||
} else if (result.getFormat() != Format::Format_0) {
|
||||
return {};
|
||||
} else if (settings.trackEstimatedTime
|
||||
!= !!(result.flags & result.kTrackEstimatedTime)) {
|
||||
return {};
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
bool BinlogWrapper::readPart() {
|
||||
if (_finished) {
|
||||
return false;
|
||||
}
|
||||
const auto no = [&] {
|
||||
finish();
|
||||
return false;
|
||||
};
|
||||
const auto offset = _binlog.offset();
|
||||
const auto left = (_till - offset);
|
||||
if (!left) {
|
||||
return no();
|
||||
}
|
||||
|
||||
if (!_part.empty() && _full.data() != _part.data()) {
|
||||
bytes::move(_full, _part);
|
||||
_part = _full.subspan(0, _part.size());
|
||||
}
|
||||
const auto amount = std::min(
|
||||
left,
|
||||
int64(_full.size() - _part.size()));
|
||||
Assert(amount > 0);
|
||||
const auto readBytes = _binlog.read(
|
||||
_full.subspan(_part.size(), amount));
|
||||
if (!readBytes) {
|
||||
return no();
|
||||
}
|
||||
_part = _full.subspan(0, _part.size() + readBytes);
|
||||
return true;
|
||||
}
|
||||
|
||||
bytes::const_span BinlogWrapper::readRecord(ReadRecordSize readRecordSize) {
|
||||
if (_finished) {
|
||||
return {};
|
||||
}
|
||||
const auto size = readRecordSize(*this, _part);
|
||||
if (size == kRecordSizeUnknown || size > _part.size()) {
|
||||
return {};
|
||||
} else if (size == kRecordSizeInvalid) {
|
||||
finish();
|
||||
_finished = _failed = true;
|
||||
return {};
|
||||
}
|
||||
Assert(size >= 0);
|
||||
const auto result = _part.subspan(0, size);
|
||||
_part = _part.subspan(size);
|
||||
return result;
|
||||
}
|
||||
|
||||
void BinlogWrapper::finish(size_type rollback) {
|
||||
Expects(rollback >= 0);
|
||||
|
||||
if (rollback > 0) {
|
||||
_failed = true;
|
||||
}
|
||||
rollback += _part.size();
|
||||
_binlog.seek(_binlog.offset() - rollback);
|
||||
}
|
||||
|
||||
} // namespace details
|
||||
} // namespace Cache
|
||||
} // namespace Storage
|
||||
|
|
@ -1,265 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include "storage/cache/storage_cache_types.h"
|
||||
#include "storage/storage_encrypted_file.h"
|
||||
#include "base/bytes.h"
|
||||
#include "base/match_method.h"
|
||||
|
||||
namespace Storage {
|
||||
namespace Cache {
|
||||
namespace details {
|
||||
|
||||
template <typename ...Records>
|
||||
class BinlogReader;
|
||||
|
||||
class BinlogWrapper {
|
||||
public:
|
||||
BinlogWrapper(File &binlog, const Settings &settings, int64 till = 0);
|
||||
|
||||
bool finished() const;
|
||||
bool failed() const;
|
||||
|
||||
static std::optional<BasicHeader> ReadHeader(
|
||||
File &binlog,
|
||||
const Settings &settings);
|
||||
|
||||
private:
|
||||
template <typename ...Records>
|
||||
friend class BinlogReader;
|
||||
|
||||
bool readPart();
|
||||
void finish(size_type rollback = 0);
|
||||
|
||||
using ReadRecordSize = size_type (*)(
|
||||
const BinlogWrapper &that,
|
||||
bytes::const_span data);
|
||||
bytes::const_span readRecord(ReadRecordSize readRecordSize);
|
||||
|
||||
File &_binlog;
|
||||
Settings _settings;
|
||||
|
||||
int64 _till = 0;
|
||||
bytes::vector _data;
|
||||
bytes::span _full;
|
||||
bytes::span _part;
|
||||
bool _finished = false;
|
||||
bool _failed = false;
|
||||
|
||||
};
|
||||
|
||||
template <typename ...Records>
|
||||
class BinlogReader {
|
||||
public:
|
||||
explicit BinlogReader(BinlogWrapper &wrapper);
|
||||
|
||||
template <typename ...Handlers>
|
||||
bool readTillEnd(Handlers &&...handlers);
|
||||
|
||||
private:
|
||||
static size_type ReadRecordSize(
|
||||
const BinlogWrapper &that,
|
||||
bytes::const_span data);
|
||||
|
||||
template <typename ...Handlers>
|
||||
bool handleRecord(bytes::const_span data, Handlers &&...handlers) const;
|
||||
|
||||
BinlogWrapper &_wrapper;
|
||||
|
||||
};
|
||||
|
||||
template <typename Record>
|
||||
struct MultiRecord {
|
||||
using true_t = char;
|
||||
using false_t = true_t(&)[2];
|
||||
static_assert(sizeof(true_t) != sizeof(false_t));
|
||||
|
||||
static false_t Check(...);
|
||||
template <typename Test, typename = typename Test::Part>
|
||||
static true_t Check(const Test&);
|
||||
|
||||
static constexpr bool Is = (sizeof(Check(std::declval<Record>()))
|
||||
== sizeof(true_t));
|
||||
};
|
||||
|
||||
template <typename ...Records>
|
||||
struct BinlogReaderRecursive {
|
||||
static void CheckSettings(const Settings &settings) {
|
||||
}
|
||||
|
||||
static size_type ReadRecordSize(
|
||||
RecordType type,
|
||||
bytes::const_span data,
|
||||
size_type partsLimit) {
|
||||
return kRecordSizeInvalid;
|
||||
}
|
||||
|
||||
template <typename ...Handlers>
|
||||
static bool HandleRecord(
|
||||
RecordType type,
|
||||
bytes::const_span data,
|
||||
Handlers &&...handlers) {
|
||||
Unexpected("Bad type in BinlogReaderRecursive::HandleRecord.");
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Record, typename ...Other>
|
||||
struct BinlogReaderRecursive<Record, Other...> {
|
||||
static void CheckSettings(const Settings &settings);
|
||||
|
||||
static size_type ReadRecordSize(
|
||||
RecordType type,
|
||||
bytes::const_span data,
|
||||
size_type partsLimit);
|
||||
|
||||
template <typename ...Handlers>
|
||||
static bool HandleRecord(
|
||||
RecordType type,
|
||||
bytes::const_span data,
|
||||
Handlers &&...handlers);
|
||||
};
|
||||
|
||||
template <typename Record, typename ...Other>
|
||||
inline void BinlogReaderRecursive<Record, Other...>::CheckSettings(
|
||||
const Settings &settings) {
|
||||
static_assert(GoodForEncryption<Record>);
|
||||
if constexpr (MultiRecord<Record>::Is) {
|
||||
using Head = Record;
|
||||
using Part = typename Record::Part;
|
||||
static_assert(GoodForEncryption<Part>);
|
||||
Assert(settings.readBlockSize
|
||||
>= (sizeof(Head)
|
||||
+ settings.maxBundledRecords * sizeof(Part)));
|
||||
} else {
|
||||
Assert(settings.readBlockSize >= sizeof(Record));
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Record, typename ...Other>
|
||||
inline size_type BinlogReaderRecursive<Record, Other...>::ReadRecordSize(
|
||||
RecordType type,
|
||||
bytes::const_span data,
|
||||
size_type partsLimit) {
|
||||
if (type != Record::kType) {
|
||||
return BinlogReaderRecursive<Other...>::ReadRecordSize(
|
||||
type,
|
||||
data,
|
||||
partsLimit);
|
||||
}
|
||||
if constexpr (MultiRecord<Record>::Is) {
|
||||
using Head = Record;
|
||||
using Part = typename Record::Part;
|
||||
|
||||
if (data.size() < sizeof(Head)) {
|
||||
return kRecordSizeUnknown;
|
||||
}
|
||||
const auto head = reinterpret_cast<const Head*>(data.data());
|
||||
const auto count = head->validateCount();
|
||||
return (count >= 0 && count <= partsLimit)
|
||||
? (sizeof(Head) + count * sizeof(Part))
|
||||
: kRecordSizeInvalid;
|
||||
} else {
|
||||
return sizeof(Record);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Record, typename ...Other>
|
||||
template <typename ...Handlers>
|
||||
inline bool BinlogReaderRecursive<Record, Other...>::HandleRecord(
|
||||
RecordType type,
|
||||
bytes::const_span data,
|
||||
Handlers &&...handlers) {
|
||||
if (type != Record::kType) {
|
||||
return BinlogReaderRecursive<Other...>::HandleRecord(
|
||||
type,
|
||||
data,
|
||||
std::forward<Handlers>(handlers)...);
|
||||
}
|
||||
if constexpr (MultiRecord<Record>::Is) {
|
||||
using Head = Record;
|
||||
using Part = typename Record::Part;
|
||||
|
||||
Assert(data.size() >= sizeof(Head));
|
||||
const auto bytes = data.data();
|
||||
const auto head = reinterpret_cast<const Head*>(bytes);
|
||||
const auto count = head->validateCount();
|
||||
Assert(data.size() == sizeof(Head) + count * sizeof(Part));
|
||||
const auto parts = gsl::make_span(
|
||||
reinterpret_cast<const Part*>(bytes + sizeof(Head)),
|
||||
count);
|
||||
auto from = std::begin(parts);
|
||||
const auto till = std::end(parts);
|
||||
const auto element = [&] {
|
||||
return (from == till) ? nullptr : &*from++;
|
||||
};
|
||||
return base::match_method2(
|
||||
*head,
|
||||
element,
|
||||
std::forward<Handlers>(handlers)...);
|
||||
} else {
|
||||
Assert(data.size() == sizeof(Record));
|
||||
return base::match_method(
|
||||
*reinterpret_cast<const Record*>(data.data()),
|
||||
std::forward<Handlers>(handlers)...);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename ...Records>
|
||||
BinlogReader<Records...>::BinlogReader(BinlogWrapper &wrapper)
|
||||
: _wrapper(wrapper) {
|
||||
BinlogReaderRecursive<Records...>::CheckSettings(_wrapper._settings);
|
||||
}
|
||||
|
||||
template <typename ...Records>
|
||||
template <typename ...Handlers>
|
||||
bool BinlogReader<Records...>::readTillEnd(Handlers &&...handlers) {
|
||||
if (!_wrapper.readPart()) {
|
||||
return true;
|
||||
}
|
||||
const auto readRecord = [&] {
|
||||
return _wrapper.readRecord(&BinlogReader::ReadRecordSize);
|
||||
};
|
||||
for (auto bytes = readRecord(); !bytes.empty(); bytes = readRecord()) {
|
||||
if (!handleRecord(bytes, std::forward<Handlers>(handlers)...)) {
|
||||
_wrapper.finish(bytes.size());
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
template <typename ...Records>
|
||||
size_type BinlogReader<Records...>::ReadRecordSize(
|
||||
const BinlogWrapper &that,
|
||||
bytes::const_span data) {
|
||||
if (data.empty()) {
|
||||
return kRecordSizeUnknown;
|
||||
}
|
||||
return BinlogReaderRecursive<Records...>::ReadRecordSize(
|
||||
static_cast<RecordType>(data[0]),
|
||||
data,
|
||||
that._settings.maxBundledRecords);
|
||||
}
|
||||
|
||||
template <typename ...Records>
|
||||
template <typename ...Handlers>
|
||||
bool BinlogReader<Records...>::handleRecord(
|
||||
bytes::const_span data,
|
||||
Handlers &&...handlers) const {
|
||||
Expects(!data.empty());
|
||||
|
||||
return BinlogReaderRecursive<Records...>::HandleRecord(
|
||||
static_cast<RecordType>(data[0]),
|
||||
data,
|
||||
std::forward<Handlers>(handlers)...);
|
||||
}
|
||||
|
||||
} // namespace details
|
||||
} // namespace Cache
|
||||
} // namespace Storage
|
||||
|
|
@ -1,111 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#include "storage/cache/storage_cache_cleaner.h"
|
||||
|
||||
#include <crl/crl.h>
|
||||
#include <QtCore/QDir>
|
||||
#include <QtCore/QFile>
|
||||
#include <unordered_map>
|
||||
#include <set>
|
||||
|
||||
namespace Storage {
|
||||
namespace Cache {
|
||||
namespace details {
|
||||
|
||||
class CleanerObject {
|
||||
public:
|
||||
CleanerObject(
|
||||
crl::weak_on_queue<CleanerObject> weak,
|
||||
const QString &base,
|
||||
base::binary_guard &&guard,
|
||||
FnMut<void(Error)> done);
|
||||
|
||||
private:
|
||||
void start();
|
||||
void scheduleNext();
|
||||
void cleanNext();
|
||||
void done();
|
||||
|
||||
crl::weak_on_queue<CleanerObject> _weak;
|
||||
QString _base, _errorPath;
|
||||
std::vector<QString> _queue;
|
||||
base::binary_guard _guard;
|
||||
FnMut<void(Error)> _done;
|
||||
|
||||
};
|
||||
|
||||
CleanerObject::CleanerObject(
|
||||
crl::weak_on_queue<CleanerObject> weak,
|
||||
const QString &base,
|
||||
base::binary_guard &&guard,
|
||||
FnMut<void(Error)> done)
|
||||
: _weak(std::move(weak))
|
||||
, _base(base)
|
||||
, _guard(std::move(guard))
|
||||
, _done(std::move(done)) {
|
||||
start();
|
||||
}
|
||||
|
||||
void CleanerObject::start() {
|
||||
const auto entries = QDir(_base).entryList(
|
||||
QDir::Dirs | QDir::NoDotAndDotDot);
|
||||
for (const auto entry : entries) {
|
||||
_queue.push_back(entry);
|
||||
}
|
||||
if (const auto version = ReadVersionValue(_base)) {
|
||||
_queue.erase(
|
||||
ranges::remove(_queue, QString::number(*version)),
|
||||
end(_queue));
|
||||
scheduleNext();
|
||||
} else {
|
||||
_errorPath = VersionFilePath(_base);
|
||||
done();
|
||||
}
|
||||
}
|
||||
|
||||
void CleanerObject::scheduleNext() {
|
||||
if (_queue.empty()) {
|
||||
done();
|
||||
return;
|
||||
}
|
||||
_weak.with([](CleanerObject &that) {
|
||||
if (that._guard) {
|
||||
that.cleanNext();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void CleanerObject::cleanNext() {
|
||||
const auto path = _base + _queue.back();
|
||||
_queue.pop_back();
|
||||
if (!QDir(path).removeRecursively()) {
|
||||
_errorPath = path;
|
||||
}
|
||||
scheduleNext();
|
||||
}
|
||||
|
||||
void CleanerObject::done() {
|
||||
if (_done) {
|
||||
_done(_errorPath.isEmpty()
|
||||
? Error::NoError()
|
||||
: Error{ Error::Type::IO, _errorPath });
|
||||
}
|
||||
}
|
||||
|
||||
Cleaner::Cleaner(
|
||||
const QString &base,
|
||||
base::binary_guard &&guard,
|
||||
FnMut<void(Error)> done)
|
||||
: _wrapped(base, std::move(guard), std::move(done)) {
|
||||
}
|
||||
|
||||
Cleaner::~Cleaner() = default;
|
||||
|
||||
} // namespace details
|
||||
} // namespace Cache
|
||||
} // namespace Storage
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include "storage/cache/storage_cache_types.h"
|
||||
#include "base/binary_guard.h"
|
||||
|
||||
namespace Storage {
|
||||
namespace Cache {
|
||||
namespace details {
|
||||
|
||||
class CleanerObject;
|
||||
|
||||
class Cleaner {
|
||||
public:
|
||||
Cleaner(
|
||||
const QString &base,
|
||||
base::binary_guard &&guard,
|
||||
FnMut<void(Error)> done);
|
||||
|
||||
~Cleaner();
|
||||
|
||||
private:
|
||||
using Implementation = details::CleanerObject;
|
||||
crl::object_on_queue<Implementation> _wrapped;
|
||||
|
||||
};
|
||||
|
||||
} // namespace details
|
||||
} // namespace Cache
|
||||
} // namespace Storage
|
||||
|
|
@ -1,434 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#include "storage/cache/storage_cache_compactor.h"
|
||||
|
||||
#include "storage/cache/storage_cache_database_object.h"
|
||||
#include "storage/cache/storage_cache_binlog_reader.h"
|
||||
#include <unordered_set>
|
||||
|
||||
namespace Storage {
|
||||
namespace Cache {
|
||||
namespace details {
|
||||
|
||||
class CompactorObject {
|
||||
public:
|
||||
using Info = Compactor::Info;
|
||||
|
||||
CompactorObject(
|
||||
crl::weak_on_queue<CompactorObject> weak,
|
||||
crl::weak_on_queue<DatabaseObject> database,
|
||||
base::binary_guard guard,
|
||||
const QString &base,
|
||||
const Settings &settings,
|
||||
EncryptionKey &&key,
|
||||
const Info &info);
|
||||
|
||||
private:
|
||||
using Entry = DatabaseObject::Entry;
|
||||
using Raw = DatabaseObject::Raw;
|
||||
using RawSpan = gsl::span<const Raw>;
|
||||
static QString CompactFilename();
|
||||
|
||||
void start();
|
||||
QString binlogPath() const;
|
||||
QString compactPath() const;
|
||||
bool openBinlog();
|
||||
bool readHeader();
|
||||
bool openCompact();
|
||||
void parseChunk();
|
||||
void fail();
|
||||
void done(int64 till);
|
||||
void finish();
|
||||
void finalize();
|
||||
|
||||
std::vector<Key> readChunk();
|
||||
bool readBlock(std::vector<Key> &result);
|
||||
void processValues(const std::vector<Raw> &values);
|
||||
|
||||
template <typename MultiRecord>
|
||||
void initList();
|
||||
RawSpan fillList(RawSpan values);
|
||||
template <typename RecordStore>
|
||||
RawSpan fillList(std::vector<RecordStore> &list, RawSpan values);
|
||||
template <typename RecordStore>
|
||||
void addListRecord(
|
||||
std::vector<RecordStore> &list,
|
||||
const Raw &raw);
|
||||
bool writeList();
|
||||
template <typename MultiRecord>
|
||||
bool writeMultiStore();
|
||||
|
||||
crl::weak_on_queue<CompactorObject> _weak;
|
||||
crl::weak_on_queue<DatabaseObject> _database;
|
||||
base::binary_guard _guard;
|
||||
QString _base;
|
||||
Settings _settings;
|
||||
EncryptionKey _key;
|
||||
BasicHeader _header;
|
||||
Info _info;
|
||||
File _binlog;
|
||||
File _compact;
|
||||
BinlogWrapper _wrapper;
|
||||
size_type _partSize = 0;
|
||||
std::unordered_set<Key> _written;
|
||||
base::variant<
|
||||
std::vector<MultiStore::Part>,
|
||||
std::vector<MultiStoreWithTime::Part>> _list;
|
||||
|
||||
};
|
||||
|
||||
CompactorObject::CompactorObject(
|
||||
crl::weak_on_queue<CompactorObject> weak,
|
||||
crl::weak_on_queue<DatabaseObject> database,
|
||||
base::binary_guard guard,
|
||||
const QString &base,
|
||||
const Settings &settings,
|
||||
EncryptionKey &&key,
|
||||
const Info &info)
|
||||
: _weak(std::move(weak))
|
||||
, _database(std::move(database))
|
||||
, _guard(std::move(guard))
|
||||
, _base(base)
|
||||
, _settings(settings)
|
||||
, _key(std::move(key))
|
||||
, _info(info)
|
||||
, _wrapper(_binlog, _settings, _info.till)
|
||||
, _partSize(_settings.maxBundledRecords) { // Perhaps a better estimate?
|
||||
Expects(_settings.compactChunkSize > 0);
|
||||
|
||||
_written.reserve(_info.keysCount);
|
||||
start();
|
||||
}
|
||||
|
||||
template <typename MultiRecord>
|
||||
void CompactorObject::initList() {
|
||||
using Part = typename MultiRecord::Part;
|
||||
auto list = std::vector<Part>();
|
||||
list.reserve(_partSize);
|
||||
_list = std::move(list);
|
||||
}
|
||||
|
||||
void CompactorObject::start() {
|
||||
if (!openBinlog() || !readHeader() || !openCompact()) {
|
||||
fail();
|
||||
}
|
||||
if (_settings.trackEstimatedTime) {
|
||||
initList<MultiStoreWithTime>();
|
||||
} else {
|
||||
initList<MultiStore>();
|
||||
}
|
||||
parseChunk();
|
||||
}
|
||||
|
||||
QString CompactorObject::CompactFilename() {
|
||||
return QStringLiteral("binlog-temp");
|
||||
}
|
||||
|
||||
QString CompactorObject::binlogPath() const {
|
||||
return _base + DatabaseObject::BinlogFilename();
|
||||
}
|
||||
|
||||
QString CompactorObject::compactPath() const {
|
||||
return _base + CompactFilename();
|
||||
}
|
||||
|
||||
bool CompactorObject::openBinlog() {
|
||||
const auto path = binlogPath();
|
||||
const auto result = _binlog.open(path, File::Mode::Read, _key);
|
||||
return (result == File::Result::Success)
|
||||
&& (_binlog.size() >= _info.till);
|
||||
}
|
||||
|
||||
bool CompactorObject::readHeader() {
|
||||
const auto header = BinlogWrapper::ReadHeader(_binlog, _settings);
|
||||
if (!header) {
|
||||
return false;
|
||||
}
|
||||
_header = *header;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool CompactorObject::openCompact() {
|
||||
const auto path = compactPath();
|
||||
const auto result = _compact.open(path, File::Mode::Write, _key);
|
||||
if (result != File::Result::Success) {
|
||||
return false;
|
||||
} else if (!_compact.write(bytes::object_as_span(&_header))) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void CompactorObject::fail() {
|
||||
_compact.close();
|
||||
QFile(compactPath()).remove();
|
||||
_database.with([](DatabaseObject &database) {
|
||||
database.compactorFail();
|
||||
});
|
||||
}
|
||||
|
||||
void CompactorObject::done(int64 till) {
|
||||
const auto path = compactPath();
|
||||
_database.with([=, good = std::move(_guard)](DatabaseObject &database) {
|
||||
if (good) {
|
||||
database.compactorDone(path, till);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void CompactorObject::finish() {
|
||||
if (writeList()) {
|
||||
finalize();
|
||||
} else {
|
||||
fail();
|
||||
}
|
||||
}
|
||||
|
||||
void CompactorObject::finalize() {
|
||||
_binlog.close();
|
||||
_compact.close();
|
||||
|
||||
auto lastCatchUp = 0;
|
||||
auto from = _info.till;
|
||||
while (true) {
|
||||
const auto till = CatchUp(
|
||||
compactPath(),
|
||||
binlogPath(),
|
||||
_key,
|
||||
from,
|
||||
_settings.readBlockSize);
|
||||
if (!till) {
|
||||
fail();
|
||||
return;
|
||||
} else if (till == from
|
||||
|| (lastCatchUp > 0 && (till - from) >= lastCatchUp)) {
|
||||
done(till);
|
||||
return;
|
||||
}
|
||||
lastCatchUp = (till - from);
|
||||
from = till;
|
||||
}
|
||||
}
|
||||
|
||||
bool CompactorObject::writeList() {
|
||||
if (_list.is<std::vector<MultiStore::Part>>()) {
|
||||
return writeMultiStore<MultiStore>();
|
||||
} else if (_list.is<std::vector<MultiStoreWithTime::Part>>()) {
|
||||
return writeMultiStore<MultiStoreWithTime>();
|
||||
} else {
|
||||
Unexpected("List type in CompactorObject::writeList.");
|
||||
}
|
||||
}
|
||||
|
||||
template <typename MultiRecord>
|
||||
bool CompactorObject::writeMultiStore() {
|
||||
using Part = typename MultiRecord::Part;
|
||||
Assert(_list.is<std::vector<Part>>());
|
||||
auto &list = _list.get_unchecked<std::vector<Part>>();
|
||||
if (list.empty()) {
|
||||
return true;
|
||||
}
|
||||
const auto guard = gsl::finally([&] { list.clear(); });
|
||||
const auto size = list.size();
|
||||
auto header = MultiRecord(size);
|
||||
if (_compact.write(bytes::object_as_span(&header))
|
||||
&& _compact.write(bytes::make_span(list))) {
|
||||
_compact.flush();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
std::vector<Key> CompactorObject::readChunk() {
|
||||
const auto limit = _settings.compactChunkSize;
|
||||
auto result = std::vector<Key>();
|
||||
while (result.size() < limit) {
|
||||
if (!readBlock(result)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
bool CompactorObject::readBlock(std::vector<Key> &result) {
|
||||
const auto push = [&](const Store &store) {
|
||||
result.push_back(store.key);
|
||||
return true;
|
||||
};
|
||||
const auto pushMulti = [&](const auto &element) {
|
||||
while (const auto record = element()) {
|
||||
push(*record);
|
||||
}
|
||||
return true;
|
||||
};
|
||||
if (_settings.trackEstimatedTime) {
|
||||
BinlogReader<
|
||||
StoreWithTime,
|
||||
MultiStoreWithTime,
|
||||
MultiRemove,
|
||||
MultiAccess> reader(_wrapper);
|
||||
return !reader.readTillEnd([&](const StoreWithTime &record) {
|
||||
return push(record);
|
||||
}, [&](const MultiStoreWithTime &header, const auto &element) {
|
||||
return pushMulti(element);
|
||||
}, [&](const MultiRemove &header, const auto &element) {
|
||||
return true;
|
||||
}, [&](const MultiAccess &header, const auto &element) {
|
||||
return true;
|
||||
});
|
||||
} else {
|
||||
BinlogReader<
|
||||
Store,
|
||||
MultiStore,
|
||||
MultiRemove> reader(_wrapper);
|
||||
return !reader.readTillEnd([&](const Store &record) {
|
||||
return push(record);
|
||||
}, [&](const MultiStore &header, const auto &element) {
|
||||
return pushMulti(element);
|
||||
}, [&](const MultiRemove &header, const auto &element) {
|
||||
return true;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void CompactorObject::parseChunk() {
|
||||
auto keys = readChunk();
|
||||
if (_wrapper.failed()) {
|
||||
fail();
|
||||
return;
|
||||
} else if (keys.empty()) {
|
||||
finish();
|
||||
return;
|
||||
}
|
||||
_database.with([
|
||||
weak = _weak,
|
||||
keys = std::move(keys)
|
||||
](DatabaseObject &database) {
|
||||
auto result = database.getManyRaw(keys);
|
||||
weak.with([result = std::move(result)](CompactorObject &that) {
|
||||
that.processValues(result);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
void CompactorObject::processValues(
|
||||
const std::vector<std::pair<Key, Entry>> &values) {
|
||||
auto left = gsl::make_span(values);
|
||||
while (true) {
|
||||
left = fillList(left);
|
||||
if (left.empty()) {
|
||||
break;
|
||||
} else if (!writeList()) {
|
||||
fail();
|
||||
return;
|
||||
}
|
||||
}
|
||||
parseChunk();
|
||||
}
|
||||
|
||||
auto CompactorObject::fillList(RawSpan values) -> RawSpan {
|
||||
return _list.match([&](auto &list) {
|
||||
return fillList(list, values);
|
||||
});
|
||||
}
|
||||
|
||||
template <typename RecordStore>
|
||||
auto CompactorObject::fillList(
|
||||
std::vector<RecordStore> &list,
|
||||
RawSpan values
|
||||
) -> RawSpan {
|
||||
const auto b = std::begin(values);
|
||||
const auto e = std::end(values);
|
||||
auto i = b;
|
||||
while (i != e && list.size() != _partSize) {
|
||||
addListRecord(list, *i++);
|
||||
}
|
||||
return values.subspan(i - b);
|
||||
}
|
||||
|
||||
template <typename RecordStore>
|
||||
void CompactorObject::addListRecord(
|
||||
std::vector<RecordStore> &list,
|
||||
const Raw &raw) {
|
||||
if (!_written.emplace(raw.first).second) {
|
||||
return;
|
||||
}
|
||||
auto record = RecordStore();
|
||||
record.key = raw.first;
|
||||
record.setSize(raw.second.size);
|
||||
record.checksum = raw.second.checksum;
|
||||
record.tag = raw.second.tag;
|
||||
record.place = raw.second.place;
|
||||
if constexpr (std::is_same_v<RecordStore, StoreWithTime>) {
|
||||
record.time.setRelative(raw.second.useTime);
|
||||
record.time.system = _info.systemTime;
|
||||
}
|
||||
list.push_back(record);
|
||||
}
|
||||
|
||||
Compactor::Compactor(
|
||||
crl::weak_on_queue<DatabaseObject> database,
|
||||
base::binary_guard guard,
|
||||
const QString &base,
|
||||
const Settings &settings,
|
||||
EncryptionKey &&key,
|
||||
const Info &info)
|
||||
: _wrapped(
|
||||
std::move(database),
|
||||
std::move(guard),
|
||||
base,
|
||||
settings,
|
||||
std::move(key),
|
||||
info) {
|
||||
}
|
||||
|
||||
Compactor::~Compactor() = default;
|
||||
|
||||
int64 CatchUp(
|
||||
const QString &compactPath,
|
||||
const QString &binlogPath,
|
||||
const EncryptionKey &key,
|
||||
int64 from,
|
||||
size_type block) {
|
||||
File binlog, compact;
|
||||
const auto result1 = binlog.open(binlogPath, File::Mode::Read, key);
|
||||
if (result1 != File::Result::Success) {
|
||||
return 0;
|
||||
}
|
||||
const auto till = binlog.size();
|
||||
if (till == from) {
|
||||
return till;
|
||||
} else if (till < from || !binlog.seek(from)) {
|
||||
return 0;
|
||||
}
|
||||
const auto result2 = compact.open(
|
||||
compactPath,
|
||||
File::Mode::ReadAppend,
|
||||
key);
|
||||
if (result2 != File::Result::Success || !compact.seek(compact.size())) {
|
||||
return 0;
|
||||
}
|
||||
auto buffer = bytes::vector(block);
|
||||
auto bytes = bytes::make_span(buffer);
|
||||
do {
|
||||
const auto left = (till - from);
|
||||
const auto limit = std::min(size_type(left), block);
|
||||
const auto read = binlog.read(bytes.subspan(0, limit));
|
||||
if (!read || read > limit) {
|
||||
return 0;
|
||||
} else if (!compact.write(bytes.subspan(0, read))) {
|
||||
return 0;
|
||||
}
|
||||
from += read;
|
||||
} while (from != till);
|
||||
return till;
|
||||
}
|
||||
|
||||
} // namespace details
|
||||
} // namespace Cache
|
||||
} // namespace Storage
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include "storage/cache/storage_cache_types.h"
|
||||
#include <crl/crl_object_on_queue.h>
|
||||
#include <base/binary_guard.h>
|
||||
|
||||
namespace Storage {
|
||||
class EncryptionKey;
|
||||
namespace Cache {
|
||||
namespace details {
|
||||
|
||||
class CompactorObject;
|
||||
class DatabaseObject;
|
||||
|
||||
class Compactor {
|
||||
public:
|
||||
struct Info {
|
||||
int64 till = 0;
|
||||
uint32 systemTime = 0;
|
||||
size_type keysCount = 0;
|
||||
};
|
||||
|
||||
Compactor(
|
||||
crl::weak_on_queue<DatabaseObject> database,
|
||||
base::binary_guard guard,
|
||||
const QString &base,
|
||||
const Settings &settings,
|
||||
EncryptionKey &&key,
|
||||
const Info &info);
|
||||
|
||||
~Compactor();
|
||||
|
||||
private:
|
||||
using Implementation = CompactorObject;
|
||||
crl::object_on_queue<Implementation> _wrapped;
|
||||
|
||||
};
|
||||
|
||||
int64 CatchUp(
|
||||
const QString &compactPath,
|
||||
const QString &binlogPath,
|
||||
const EncryptionKey &key,
|
||||
int64 from,
|
||||
size_type block);
|
||||
|
||||
} // namespace details
|
||||
} // namespace Cache
|
||||
} // namespace Storage
|
||||
|
|
@ -1,203 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#include "storage/cache/storage_cache_database.h"
|
||||
|
||||
#include "storage/cache/storage_cache_database_object.h"
|
||||
|
||||
namespace Storage {
|
||||
namespace Cache {
|
||||
|
||||
Database::Database(const QString &path, const Settings &settings)
|
||||
: _wrapped(path, settings) {
|
||||
}
|
||||
|
||||
void Database::reconfigure(const Settings &settings) {
|
||||
_wrapped.with([settings](Implementation &unwrapped) mutable {
|
||||
unwrapped.reconfigure(settings);
|
||||
});
|
||||
}
|
||||
|
||||
void Database::updateSettings(const SettingsUpdate &update) {
|
||||
_wrapped.with([update](Implementation &unwrapped) mutable {
|
||||
unwrapped.updateSettings(update);
|
||||
});
|
||||
}
|
||||
|
||||
void Database::open(EncryptionKey &&key, FnMut<void(Error)> &&done) {
|
||||
_wrapped.with([
|
||||
key = std::move(key),
|
||||
done = std::move(done)
|
||||
](Implementation &unwrapped) mutable {
|
||||
unwrapped.open(std::move(key), std::move(done));
|
||||
});
|
||||
}
|
||||
|
||||
void Database::close(FnMut<void()> &&done) {
|
||||
_wrapped.with([
|
||||
done = std::move(done)
|
||||
](Implementation &unwrapped) mutable {
|
||||
unwrapped.close(std::move(done));
|
||||
});
|
||||
}
|
||||
|
||||
void Database::waitForCleaner(FnMut<void()> &&done) {
|
||||
_wrapped.with([
|
||||
done = std::move(done)
|
||||
](Implementation &unwrapped) mutable {
|
||||
unwrapped.waitForCleaner(std::move(done));
|
||||
});
|
||||
}
|
||||
|
||||
void Database::put(
|
||||
const Key &key,
|
||||
QByteArray &&value,
|
||||
FnMut<void(Error)> &&done) {
|
||||
return put(key, TaggedValue(std::move(value), 0), std::move(done));
|
||||
}
|
||||
|
||||
void Database::get(const Key &key, FnMut<void(QByteArray&&)> &&done) {
|
||||
if (done) {
|
||||
auto untag = [done = std::move(done)](TaggedValue &&value) mutable {
|
||||
done(std::move(value.bytes));
|
||||
};
|
||||
getWithTag(key, std::move(untag));
|
||||
} else {
|
||||
getWithTag(key, nullptr);
|
||||
}
|
||||
}
|
||||
|
||||
void Database::remove(const Key &key, FnMut<void(Error)> &&done) {
|
||||
_wrapped.with([
|
||||
key,
|
||||
done = std::move(done)
|
||||
](Implementation &unwrapped) mutable {
|
||||
unwrapped.remove(key, std::move(done));
|
||||
});
|
||||
}
|
||||
|
||||
void Database::putIfEmpty(
|
||||
const Key &key,
|
||||
QByteArray &&value,
|
||||
FnMut<void(Error)> &&done) {
|
||||
return putIfEmpty(
|
||||
key,
|
||||
TaggedValue(std::move(value), 0),
|
||||
std::move(done));
|
||||
}
|
||||
|
||||
void Database::copyIfEmpty(
|
||||
const Key &from,
|
||||
const Key &to,
|
||||
FnMut<void(Error)> &&done) {
|
||||
_wrapped.with([
|
||||
from,
|
||||
to,
|
||||
done = std::move(done)
|
||||
](Implementation &unwrapped) mutable {
|
||||
unwrapped.copyIfEmpty(from, to, std::move(done));
|
||||
});
|
||||
}
|
||||
|
||||
void Database::moveIfEmpty(
|
||||
const Key &from,
|
||||
const Key &to,
|
||||
FnMut<void(Error)> &&done) {
|
||||
_wrapped.with([
|
||||
from,
|
||||
to,
|
||||
done = std::move(done)
|
||||
](Implementation &unwrapped) mutable {
|
||||
unwrapped.moveIfEmpty(from, to, std::move(done));
|
||||
});
|
||||
}
|
||||
|
||||
void Database::put(
|
||||
const Key &key,
|
||||
TaggedValue &&value,
|
||||
FnMut<void(Error)> &&done) {
|
||||
_wrapped.with([
|
||||
key,
|
||||
value = std::move(value),
|
||||
done = std::move(done)
|
||||
](Implementation &unwrapped) mutable {
|
||||
unwrapped.put(key, std::move(value), std::move(done));
|
||||
});
|
||||
}
|
||||
|
||||
void Database::putIfEmpty(
|
||||
const Key &key,
|
||||
TaggedValue &&value,
|
||||
FnMut<void(Error)> &&done) {
|
||||
_wrapped.with([
|
||||
key,
|
||||
value = std::move(value),
|
||||
done = std::move(done)
|
||||
](Implementation &unwrapped) mutable {
|
||||
unwrapped.putIfEmpty(key, std::move(value), std::move(done));
|
||||
});
|
||||
}
|
||||
|
||||
void Database::getWithTag(
|
||||
const Key &key,
|
||||
FnMut<void(TaggedValue&&)> &&done) {
|
||||
_wrapped.with([
|
||||
key,
|
||||
done = std::move(done)
|
||||
](Implementation &unwrapped) mutable {
|
||||
unwrapped.get(key, std::move(done));
|
||||
});
|
||||
}
|
||||
|
||||
void Database::getWithSizes(
|
||||
const Key &key,
|
||||
std::vector<Key> &&keys,
|
||||
FnMut<void(QByteArray&&, std::vector<int>&&)> &&done) {
|
||||
_wrapped.with([
|
||||
key,
|
||||
keys = std::move(keys),
|
||||
done = std::move(done)
|
||||
](Implementation &unwrapped) mutable {
|
||||
unwrapped.getWithSizes(key, std::move(keys), std::move(done));
|
||||
});
|
||||
}
|
||||
|
||||
auto Database::statsOnMain() const -> rpl::producer<Stats> {
|
||||
return _wrapped.producer_on_main([](const Implementation &unwrapped) {
|
||||
return unwrapped.stats();
|
||||
});
|
||||
}
|
||||
|
||||
void Database::clear(FnMut<void(Error)> &&done) {
|
||||
_wrapped.with([
|
||||
done = std::move(done)
|
||||
](Implementation &unwrapped) mutable {
|
||||
unwrapped.clear(std::move(done));
|
||||
});
|
||||
}
|
||||
|
||||
void Database::clearByTag(uint8 tag, FnMut<void(Error)> &&done) {
|
||||
_wrapped.with([
|
||||
tag,
|
||||
done = std::move(done)
|
||||
](Implementation &unwrapped) mutable {
|
||||
unwrapped.clearByTag(tag, std::move(done));
|
||||
});
|
||||
}
|
||||
|
||||
void Database::sync() {
|
||||
auto semaphore = crl::semaphore();
|
||||
_wrapped.with([&](Implementation &) {
|
||||
semaphore.release();
|
||||
});
|
||||
semaphore.acquire();
|
||||
}
|
||||
|
||||
Database::~Database() = default;
|
||||
|
||||
} // namespace Cache
|
||||
} // namespace Storage
|
||||
|
|
@ -1,91 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include "storage/cache/storage_cache_types.h"
|
||||
#include "base/basic_types.h"
|
||||
#include <crl/crl_object_on_queue.h>
|
||||
#include <crl/crl_time.h>
|
||||
#include <rpl/producer.h>
|
||||
#include <QtCore/QString>
|
||||
|
||||
namespace Storage {
|
||||
class EncryptionKey;
|
||||
namespace Cache {
|
||||
namespace details {
|
||||
class DatabaseObject;
|
||||
} // namespace details
|
||||
|
||||
class Database {
|
||||
public:
|
||||
using Settings = details::Settings;
|
||||
using SettingsUpdate = details::SettingsUpdate;
|
||||
Database(const QString &path, const Settings &settings);
|
||||
|
||||
void reconfigure(const Settings &settings);
|
||||
void updateSettings(const SettingsUpdate &update);
|
||||
|
||||
void open(EncryptionKey &&key, FnMut<void(Error)> &&done = nullptr);
|
||||
void close(FnMut<void()> &&done = nullptr);
|
||||
|
||||
void put(
|
||||
const Key &key,
|
||||
QByteArray &&value,
|
||||
FnMut<void(Error)> &&done = nullptr);
|
||||
void get(const Key &key, FnMut<void(QByteArray&&)> &&done);
|
||||
void remove(const Key &key, FnMut<void(Error)> &&done = nullptr);
|
||||
|
||||
void putIfEmpty(
|
||||
const Key &key,
|
||||
QByteArray &&value,
|
||||
FnMut<void(Error)> &&done = nullptr);
|
||||
void copyIfEmpty(
|
||||
const Key &from,
|
||||
const Key &to,
|
||||
FnMut<void(Error)> &&done = nullptr);
|
||||
void moveIfEmpty(
|
||||
const Key &from,
|
||||
const Key &to,
|
||||
FnMut<void(Error)> &&done = nullptr);
|
||||
|
||||
using TaggedValue = details::TaggedValue;
|
||||
void put(
|
||||
const Key &key,
|
||||
TaggedValue &&value,
|
||||
FnMut<void(Error)> &&done = nullptr);
|
||||
void putIfEmpty(
|
||||
const Key &key,
|
||||
TaggedValue &&value,
|
||||
FnMut<void(Error)> &&done = nullptr);
|
||||
void getWithTag(const Key &key, FnMut<void(TaggedValue&&)> &&done);
|
||||
|
||||
void getWithSizes(
|
||||
const Key &key,
|
||||
std::vector<Key> &&keys,
|
||||
FnMut<void(QByteArray&&, std::vector<int>&&)> &&done);
|
||||
|
||||
using Stats = details::Stats;
|
||||
using TaggedSummary = details::TaggedSummary;
|
||||
rpl::producer<Stats> statsOnMain() const;
|
||||
|
||||
void clear(FnMut<void(Error)> &&done = nullptr);
|
||||
void clearByTag(uint8 tag, FnMut<void(Error)> &&done = nullptr);
|
||||
void waitForCleaner(FnMut<void()> &&done = nullptr);
|
||||
|
||||
void sync();
|
||||
|
||||
~Database();
|
||||
|
||||
private:
|
||||
using Implementation = details::DatabaseObject;
|
||||
crl::object_on_queue<Implementation> _wrapped;
|
||||
|
||||
};
|
||||
|
||||
} // namespace Cache
|
||||
} // namespace Storage
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,256 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include "storage/cache/storage_cache_database.h"
|
||||
#include "storage/storage_encrypted_file.h"
|
||||
#include "base/binary_guard.h"
|
||||
#include "base/concurrent_timer.h"
|
||||
#include "base/bytes.h"
|
||||
#include "base/flat_set.h"
|
||||
#include <set>
|
||||
#include <rpl/event_stream.h>
|
||||
|
||||
namespace Storage {
|
||||
namespace Cache {
|
||||
namespace details {
|
||||
|
||||
class Cleaner;
|
||||
class Compactor;
|
||||
|
||||
class DatabaseObject {
|
||||
public:
|
||||
using Settings = Cache::Database::Settings;
|
||||
DatabaseObject(
|
||||
crl::weak_on_queue<DatabaseObject> weak,
|
||||
const QString &path,
|
||||
const Settings &settings);
|
||||
void reconfigure(const Settings &settings);
|
||||
void updateSettings(const SettingsUpdate &update);
|
||||
|
||||
void open(EncryptionKey &&key, FnMut<void(Error)> &&done);
|
||||
void close(FnMut<void()> &&done);
|
||||
|
||||
void put(
|
||||
const Key &key,
|
||||
TaggedValue &&value,
|
||||
FnMut<void(Error)> &&done);
|
||||
void get(const Key &key, FnMut<void(TaggedValue&&)> &&done);
|
||||
void remove(const Key &key, FnMut<void(Error)> &&done);
|
||||
|
||||
void putIfEmpty(
|
||||
const Key &key,
|
||||
TaggedValue &&value,
|
||||
FnMut<void(Error)> &&done);
|
||||
void copyIfEmpty(
|
||||
const Key &from,
|
||||
const Key &to,
|
||||
FnMut<void(Error)> &&done);
|
||||
void moveIfEmpty(
|
||||
const Key &from,
|
||||
const Key &to,
|
||||
FnMut<void(Error)> &&done);
|
||||
|
||||
void getWithSizes(
|
||||
const Key &key,
|
||||
std::vector<Key> &&keys,
|
||||
FnMut<void(QByteArray&&, std::vector<int>&&)> &&done);
|
||||
|
||||
rpl::producer<Stats> stats() const;
|
||||
|
||||
void clear(FnMut<void(Error)> &&done);
|
||||
void clearByTag(uint8 tag, FnMut<void(Error)> &&done);
|
||||
void waitForCleaner(FnMut<void()> &&done);
|
||||
|
||||
static QString BinlogFilename();
|
||||
static QString CompactReadyFilename();
|
||||
|
||||
void compactorDone(const QString &path, int64 originalReadTill);
|
||||
void compactorFail();
|
||||
|
||||
struct Entry {
|
||||
Entry() = default;
|
||||
Entry(
|
||||
PlaceId place,
|
||||
uint8 tag,
|
||||
uint32 checksum,
|
||||
size_type size,
|
||||
uint64 useTime);
|
||||
|
||||
uint64 useTime = 0;
|
||||
size_type size = 0;
|
||||
uint32 checksum = 0;
|
||||
PlaceId place = { { 0 } };
|
||||
uint8 tag = 0;
|
||||
};
|
||||
using Raw = std::pair<Key, Entry>;
|
||||
std::vector<Raw> getManyRaw(const std::vector<Key> &keys) const;
|
||||
|
||||
~DatabaseObject();
|
||||
|
||||
private:
|
||||
struct CleanerWrap {
|
||||
std::unique_ptr<Cleaner> object;
|
||||
base::binary_guard guard;
|
||||
FnMut<void()> done;
|
||||
};
|
||||
struct CompactorWrap {
|
||||
std::unique_ptr<Compactor> object;
|
||||
int64 excessLength = 0;
|
||||
crl::time nextAttempt = 0;
|
||||
crl::time delayAfterFailure = 10 * crl::time(1000);
|
||||
base::binary_guard guard;
|
||||
};
|
||||
using Map = std::unordered_map<Key, Entry>;
|
||||
|
||||
template <typename Callback, typename ...Args>
|
||||
void invokeCallback(Callback &&callback, Args &&...args) const;
|
||||
|
||||
Error ioError(const QString &path) const;
|
||||
|
||||
void checkSettings();
|
||||
QString computePath(Version version) const;
|
||||
QString binlogPath(Version version) const;
|
||||
QString binlogPath() const;
|
||||
QString compactReadyPath(Version version) const;
|
||||
QString compactReadyPath() const;
|
||||
Error openSomeBinlog(EncryptionKey &&key);
|
||||
Error openNewBinlog(EncryptionKey &key);
|
||||
File::Result openBinlog(
|
||||
Version version,
|
||||
File::Mode mode,
|
||||
EncryptionKey &key);
|
||||
bool readHeader();
|
||||
bool writeHeader();
|
||||
|
||||
void readBinlog();
|
||||
template <typename Reader, typename ...Handlers>
|
||||
void readBinlogHelper(Reader &reader, Handlers &&...handlers);
|
||||
template <typename Record, typename Postprocess>
|
||||
bool processRecordStoreGeneric(
|
||||
const Record *record,
|
||||
Postprocess &&postprocess);
|
||||
bool processRecordStore(const Store *record, std::is_class<Store>);
|
||||
bool processRecordStore(
|
||||
const StoreWithTime *record,
|
||||
std::is_class<StoreWithTime>);
|
||||
template <typename Record, typename GetElement>
|
||||
bool processRecordMultiStore(
|
||||
const Record &header,
|
||||
const GetElement &element);
|
||||
template <typename GetElement>
|
||||
bool processRecordMultiRemove(
|
||||
const MultiRemove &header,
|
||||
const GetElement &element);
|
||||
template <typename GetElement>
|
||||
bool processRecordMultiAccess(
|
||||
const MultiAccess &header,
|
||||
const GetElement &element);
|
||||
|
||||
void optimize();
|
||||
void checkCompactor();
|
||||
void adjustRelativeTime();
|
||||
bool startDelayedPruning();
|
||||
uint64 countRelativeTime() const;
|
||||
EstimatedTimePoint countTimePoint() const;
|
||||
void applyTimePoint(EstimatedTimePoint time);
|
||||
|
||||
uint64 pruneBeforeTime() const;
|
||||
void prune();
|
||||
void collectTimeStale(
|
||||
base::flat_set<Key> &stale,
|
||||
int64 &staleTotalSize);
|
||||
void collectSizeStale(
|
||||
base::flat_set<Key> &stale,
|
||||
int64 &staleTotalSize);
|
||||
void startStaleClear();
|
||||
void clearStaleNow(const base::flat_set<Key> &stale);
|
||||
void clearStaleChunkDelayed();
|
||||
void clearStaleChunk();
|
||||
|
||||
void updateStats(const Entry &was, const Entry &now);
|
||||
Stats collectStats() const;
|
||||
void pushStatsDelayed();
|
||||
void pushStats();
|
||||
|
||||
void setMapEntry(const Key &key, Entry &&entry);
|
||||
void eraseMapEntry(const Map::const_iterator &i);
|
||||
void recordEntryAccess(const Key &key);
|
||||
QByteArray readValueData(PlaceId place, size_type size) const;
|
||||
|
||||
Version findAvailableVersion() const;
|
||||
QString versionPath() const;
|
||||
bool writeVersion(Version version);
|
||||
Version readVersion() const;
|
||||
|
||||
QString placePath(PlaceId place) const;
|
||||
bool isFreePlace(PlaceId place) const;
|
||||
|
||||
template <typename StoreRecord>
|
||||
std::optional<QString> writeKeyPlaceGeneric(
|
||||
StoreRecord &&record,
|
||||
const Key &key,
|
||||
const TaggedValue &value,
|
||||
uint32 checksum);
|
||||
std::optional<QString> writeKeyPlace(
|
||||
const Key &key,
|
||||
const TaggedValue &value,
|
||||
uint32 checksum);
|
||||
template <typename StoreRecord>
|
||||
Error writeExistingPlaceGeneric(
|
||||
StoreRecord &&record,
|
||||
const Key &key,
|
||||
const Entry &entry);
|
||||
Error writeExistingPlace(
|
||||
const Key &key,
|
||||
const Entry &entry);
|
||||
void writeMultiRemoveLazy();
|
||||
Error writeMultiRemove();
|
||||
void writeMultiAccessLazy();
|
||||
Error writeMultiAccess();
|
||||
Error writeMultiAccessBlock();
|
||||
void writeBundlesLazy();
|
||||
void writeBundles();
|
||||
|
||||
void createCleaner();
|
||||
void cleanerDone(Error error);
|
||||
void clearState();
|
||||
|
||||
crl::weak_on_queue<DatabaseObject> _weak;
|
||||
QString _base, _path;
|
||||
Settings _settings;
|
||||
EncryptionKey _key;
|
||||
File _binlog;
|
||||
Map _map;
|
||||
std::set<Key> _removing;
|
||||
std::set<Key> _accessed;
|
||||
std::vector<Key> _stale;
|
||||
|
||||
EstimatedTimePoint _time;
|
||||
|
||||
int64 _binlogExcessLength = 0;
|
||||
int64 _totalSize = 0;
|
||||
uint64 _minimalEntryTime = 0;
|
||||
size_type _entriesWithMinimalTimeCount = 0;
|
||||
|
||||
base::flat_map<uint8, TaggedSummary> _taggedStats;
|
||||
rpl::event_stream<Stats> _stats;
|
||||
bool _pushingStats = false;
|
||||
bool _clearingStale = false;
|
||||
|
||||
base::ConcurrentTimer _writeBundlesTimer;
|
||||
base::ConcurrentTimer _pruneTimer;
|
||||
|
||||
CleanerWrap _cleaner;
|
||||
CompactorWrap _compactor;
|
||||
|
||||
};
|
||||
|
||||
} // namespace details
|
||||
} // namespace Cache
|
||||
} // namespace Storage
|
||||
|
|
@ -1,740 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#include "catch.hpp"
|
||||
|
||||
#include "storage/cache/storage_cache_database.h"
|
||||
#include "storage/storage_encryption.h"
|
||||
#include "storage/storage_encrypted_file.h"
|
||||
#include "base/concurrent_timer.h"
|
||||
#include <crl/crl.h>
|
||||
#include <QtCore/QFile>
|
||||
#include <QtWidgets/QApplication>
|
||||
#include <thread>
|
||||
|
||||
using namespace Storage::Cache;
|
||||
|
||||
const auto DisableLimitsTests = false;
|
||||
const auto DisableCompactTests = false;
|
||||
const auto DisableLargeTest = true;
|
||||
|
||||
const auto key = Storage::EncryptionKey(bytes::make_vector(
|
||||
bytes::make_span("\
|
||||
abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\
|
||||
abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\
|
||||
abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\
|
||||
abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\
|
||||
").subspan(0, Storage::EncryptionKey::kSize)));
|
||||
|
||||
const auto name = QString("test.db");
|
||||
|
||||
const auto SmallSleep = [] {
|
||||
static auto SleepTime = 0;
|
||||
if (SleepTime > 5000) {
|
||||
return false;
|
||||
}
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(10));
|
||||
SleepTime += 10;
|
||||
return true;
|
||||
};
|
||||
|
||||
QString GetBinlogPath() {
|
||||
using namespace Storage;
|
||||
|
||||
QFile versionFile(name + "/version");
|
||||
while (!versionFile.open(QIODevice::ReadOnly)) {
|
||||
if (!SmallSleep()) {
|
||||
return QString();
|
||||
}
|
||||
}
|
||||
const auto bytes = versionFile.readAll();
|
||||
if (bytes.size() != 4) {
|
||||
return QString();
|
||||
}
|
||||
const auto version = *reinterpret_cast<const int32*>(bytes.data());
|
||||
return name + '/' + QString::number(version) + "/binlog";
|
||||
}
|
||||
|
||||
const auto Test1 = [] {
|
||||
static auto result = QByteArray("testbytetestbyt");
|
||||
return result;
|
||||
};
|
||||
const auto Test2 = [] {
|
||||
static auto result = QByteArray("bytetestbytetestb");
|
||||
return result;
|
||||
};
|
||||
|
||||
crl::semaphore Semaphore;
|
||||
|
||||
auto Result = Error();
|
||||
const auto GetResult = [](Error error) {
|
||||
Result = error;
|
||||
Semaphore.release();
|
||||
};
|
||||
|
||||
auto Value = QByteArray();
|
||||
const auto GetValue = [](QByteArray value) {
|
||||
Value = value;
|
||||
Semaphore.release();
|
||||
};
|
||||
|
||||
auto ValueWithTag = Database::TaggedValue();
|
||||
const auto GetValueWithTag = [](Database::TaggedValue value) {
|
||||
ValueWithTag = value;
|
||||
Semaphore.release();
|
||||
};
|
||||
|
||||
Error Open(Database &db, const Storage::EncryptionKey &key) {
|
||||
db.open(base::duplicate(key), GetResult);
|
||||
Semaphore.acquire();
|
||||
return Result;
|
||||
}
|
||||
|
||||
void Close(Database &db) {
|
||||
db.close([&] { Semaphore.release(); });
|
||||
Semaphore.acquire();
|
||||
}
|
||||
|
||||
Error Clear(Database &db) {
|
||||
db.clear(GetResult);
|
||||
Semaphore.acquire();
|
||||
return Result;
|
||||
}
|
||||
|
||||
QByteArray Get(Database &db, const Key &key) {
|
||||
db.get(key, GetValue);
|
||||
Semaphore.acquire();
|
||||
return Value;
|
||||
}
|
||||
|
||||
Database::TaggedValue GetWithTag(Database &db, const Key &key) {
|
||||
db.getWithTag(key, GetValueWithTag);
|
||||
Semaphore.acquire();
|
||||
return ValueWithTag;
|
||||
}
|
||||
|
||||
Error Put(Database &db, const Key &key, QByteArray &&value) {
|
||||
db.put(key, std::move(value), GetResult);
|
||||
Semaphore.acquire();
|
||||
return Result;
|
||||
}
|
||||
|
||||
Error Put(Database &db, const Key &key, Database::TaggedValue &&value) {
|
||||
db.put(key, std::move(value), GetResult);
|
||||
Semaphore.acquire();
|
||||
return Result;
|
||||
}
|
||||
|
||||
Error PutIfEmpty(Database &db, const Key &key, QByteArray &&value) {
|
||||
db.putIfEmpty(key, std::move(value), GetResult);
|
||||
Semaphore.acquire();
|
||||
return Result;
|
||||
}
|
||||
|
||||
Error CopyIfEmpty(Database &db, const Key &from, const Key &to) {
|
||||
db.copyIfEmpty(from, to, GetResult);
|
||||
Semaphore.acquire();
|
||||
return Result;
|
||||
}
|
||||
|
||||
Error MoveIfEmpty(Database &db, const Key &from, const Key &to) {
|
||||
db.moveIfEmpty(from, to, GetResult);
|
||||
Semaphore.acquire();
|
||||
return Result;
|
||||
}
|
||||
|
||||
void Remove(Database &db, const Key &key) {
|
||||
db.remove(key, [&](Error) { Semaphore.release(); });
|
||||
Semaphore.acquire();
|
||||
}
|
||||
|
||||
Error ClearByTag(Database &db, uint8 tag) {
|
||||
db.clearByTag(tag, GetResult);
|
||||
Semaphore.acquire();
|
||||
return Result;
|
||||
}
|
||||
|
||||
const auto Settings = [] {
|
||||
auto result = Database::Settings();
|
||||
result.trackEstimatedTime = false;
|
||||
result.writeBundleDelay = 1 * crl::time(1000);
|
||||
result.pruneTimeout = 1 * crl::time(1500);
|
||||
result.maxDataSize = 20;
|
||||
return result;
|
||||
}();
|
||||
|
||||
const auto AdvanceTime = [](int32 seconds) {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(1000) * seconds);
|
||||
};
|
||||
|
||||
TEST_CASE("init timers", "[storage_cache_database]") {
|
||||
static auto init = [] {
|
||||
int argc = 0;
|
||||
char **argv = nullptr;
|
||||
static QCoreApplication application(argc, argv);
|
||||
static base::ConcurrentTimerEnvironment environment;
|
||||
return true;
|
||||
}();
|
||||
}
|
||||
|
||||
TEST_CASE("compacting db", "[storage_cache_database]") {
|
||||
if (DisableCompactTests || !DisableLargeTest) {
|
||||
return;
|
||||
}
|
||||
const auto write = [](Database &db, uint32 from, uint32 till, QByteArray base) {
|
||||
for (auto i = from; i != till; ++i) {
|
||||
auto value = base;
|
||||
value[0] = char('A') + i;
|
||||
const auto result = Put(db, Key{ i, i + 1 }, std::move(value));
|
||||
REQUIRE(result.type == Error::Type::None);
|
||||
}
|
||||
};
|
||||
const auto put = [&](Database &db, uint32 from, uint32 till) {
|
||||
write(db, from, till, Test1());
|
||||
};
|
||||
const auto reput = [&](Database &db, uint32 from, uint32 till) {
|
||||
write(db, from, till, Test2());
|
||||
};
|
||||
const auto remove = [](Database &db, uint32 from, uint32 till) {
|
||||
for (auto i = from; i != till; ++i) {
|
||||
Remove(db, Key{ i, i + 1 });
|
||||
}
|
||||
};
|
||||
const auto get = [](Database &db, uint32 from, uint32 till) {
|
||||
for (auto i = from; i != till; ++i) {
|
||||
db.get(Key{ i, i + 1 }, nullptr);
|
||||
}
|
||||
};
|
||||
const auto check = [](Database &db, uint32 from, uint32 till, QByteArray base) {
|
||||
for (auto i = from; i != till; ++i) {
|
||||
auto value = base;
|
||||
if (!value.isEmpty()) {
|
||||
value[0] = char('A') + i;
|
||||
}
|
||||
const auto result = Get(db, Key{ i, i + 1 });
|
||||
REQUIRE((result == value));
|
||||
}
|
||||
};
|
||||
SECTION("simple compact with min size") {
|
||||
auto settings = Settings;
|
||||
settings.writeBundleDelay = crl::time(100);
|
||||
settings.readBlockSize = 512;
|
||||
settings.maxBundledRecords = 5;
|
||||
settings.compactAfterExcess = (3 * (16 * 5 + 16) + 15 * 32) / 2;
|
||||
settings.compactAfterFullSize = (sizeof(details::BasicHeader)
|
||||
+ 40 * 32) / 2
|
||||
+ settings.compactAfterExcess;
|
||||
Database db(name, settings);
|
||||
|
||||
REQUIRE(Clear(db).type == Error::Type::None);
|
||||
REQUIRE(Open(db, key).type == Error::Type::None);
|
||||
put(db, 0, 30);
|
||||
remove(db, 0, 15);
|
||||
put(db, 30, 40);
|
||||
reput(db, 15, 29);
|
||||
AdvanceTime(1);
|
||||
const auto path = GetBinlogPath();
|
||||
const auto size = QFile(path).size();
|
||||
reput(db, 29, 30); // starts compactor
|
||||
AdvanceTime(2);
|
||||
REQUIRE(QFile(path).size() < size);
|
||||
remove(db, 30, 35);
|
||||
reput(db, 35, 37);
|
||||
put(db, 15, 20);
|
||||
put(db, 40, 45);
|
||||
|
||||
const auto fullcheck = [&] {
|
||||
check(db, 0, 15, {});
|
||||
check(db, 15, 20, Test1());
|
||||
check(db, 20, 30, Test2());
|
||||
check(db, 30, 35, {});
|
||||
check(db, 35, 37, Test2());
|
||||
check(db, 37, 45, Test1());
|
||||
};
|
||||
fullcheck();
|
||||
Close(db);
|
||||
|
||||
REQUIRE(Open(db, key).type == Error::Type::None);
|
||||
fullcheck();
|
||||
Close(db);
|
||||
}
|
||||
SECTION("simple compact without min size") {
|
||||
auto settings = Settings;
|
||||
settings.writeBundleDelay = crl::time(100);
|
||||
settings.readBlockSize = 512;
|
||||
settings.maxBundledRecords = 5;
|
||||
settings.compactAfterExcess = 3 * (16 * 5 + 16) + 15 * 32;
|
||||
Database db(name, settings);
|
||||
|
||||
REQUIRE(Clear(db).type == Error::Type::None);
|
||||
REQUIRE(Open(db, key).type == Error::Type::None);
|
||||
put(db, 0, 30);
|
||||
remove(db, 0, 15);
|
||||
put(db, 30, 40);
|
||||
reput(db, 15, 29);
|
||||
AdvanceTime(1);
|
||||
const auto path = GetBinlogPath();
|
||||
const auto size = QFile(path).size();
|
||||
reput(db, 29, 30); // starts compactor
|
||||
AdvanceTime(2);
|
||||
REQUIRE(QFile(path).size() < size);
|
||||
remove(db, 30, 35);
|
||||
reput(db, 35, 37);
|
||||
put(db, 15, 20);
|
||||
put(db, 40, 45);
|
||||
|
||||
const auto fullcheck = [&] {
|
||||
check(db, 0, 15, {});
|
||||
check(db, 15, 20, Test1());
|
||||
check(db, 20, 30, Test2());
|
||||
check(db, 30, 35, {});
|
||||
check(db, 35, 37, Test2());
|
||||
check(db, 37, 45, Test1());
|
||||
};
|
||||
fullcheck();
|
||||
Close(db);
|
||||
|
||||
REQUIRE(Open(db, key).type == Error::Type::None);
|
||||
fullcheck();
|
||||
Close(db);
|
||||
}
|
||||
SECTION("double compact") {
|
||||
auto settings = Settings;
|
||||
settings.writeBundleDelay = crl::time(100);
|
||||
settings.readBlockSize = 512;
|
||||
settings.maxBundledRecords = 5;
|
||||
settings.compactAfterExcess = 3 * (16 * 5 + 16) + 15 * 32;
|
||||
Database db(name, settings);
|
||||
|
||||
REQUIRE(Clear(db).type == Error::Type::None);
|
||||
REQUIRE(Open(db, key).type == Error::Type::None);
|
||||
put(db, 0, 30);
|
||||
remove(db, 0, 15);
|
||||
reput(db, 15, 29);
|
||||
AdvanceTime(1);
|
||||
const auto path = GetBinlogPath();
|
||||
const auto size1 = QFile(path).size();
|
||||
reput(db, 29, 30); // starts compactor
|
||||
AdvanceTime(2);
|
||||
REQUIRE(QFile(path).size() < size1);
|
||||
put(db, 30, 45);
|
||||
remove(db, 20, 35);
|
||||
put(db, 15, 20);
|
||||
reput(db, 35, 44);
|
||||
const auto size2 = QFile(path).size();
|
||||
reput(db, 44, 45); // starts compactor
|
||||
AdvanceTime(2);
|
||||
const auto after = QFile(path).size();
|
||||
REQUIRE(after < size1);
|
||||
REQUIRE(after < size2);
|
||||
const auto fullcheck = [&] {
|
||||
check(db, 0, 15, {});
|
||||
check(db, 15, 20, Test1());
|
||||
check(db, 20, 35, {});
|
||||
check(db, 35, 45, Test2());
|
||||
};
|
||||
fullcheck();
|
||||
Close(db);
|
||||
|
||||
REQUIRE(Open(db, key).type == Error::Type::None);
|
||||
fullcheck();
|
||||
Close(db);
|
||||
}
|
||||
SECTION("time tracking compact") {
|
||||
auto settings = Settings;
|
||||
settings.writeBundleDelay = crl::time(100);
|
||||
settings.trackEstimatedTime = true;
|
||||
settings.readBlockSize = 512;
|
||||
settings.maxBundledRecords = 5;
|
||||
settings.compactAfterExcess = 6 * (16 * 5 + 16)
|
||||
+ 3 * (16 * 5 + 16)
|
||||
+ 15 * 48
|
||||
+ 3 * (16 * 5 + 16)
|
||||
+ (16 * 1 + 16);
|
||||
Database db(name, settings);
|
||||
|
||||
REQUIRE(Clear(db).type == Error::Type::None);
|
||||
REQUIRE(Open(db, key).type == Error::Type::None);
|
||||
put(db, 0, 30);
|
||||
get(db, 0, 30);
|
||||
//AdvanceTime(1); get's will be written instantly becase !(30 % 5)
|
||||
remove(db, 0, 15);
|
||||
reput(db, 15, 30);
|
||||
get(db, 0, 30);
|
||||
AdvanceTime(1);
|
||||
const auto path = GetBinlogPath();
|
||||
const auto size = QFile(path).size();
|
||||
get(db, 29, 30); // starts compactor delayed
|
||||
AdvanceTime(2);
|
||||
REQUIRE(QFile(path).size() < size);
|
||||
const auto fullcheck = [&] {
|
||||
check(db, 15, 30, Test2());
|
||||
};
|
||||
fullcheck();
|
||||
Close(db);
|
||||
|
||||
REQUIRE(Open(db, key).type == Error::Type::None);
|
||||
fullcheck();
|
||||
Close(db);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("encrypted cache db", "[storage_cache_database]") {
|
||||
if (!DisableLargeTest) {
|
||||
return;
|
||||
}
|
||||
SECTION("writing db") {
|
||||
Database db(name, Settings);
|
||||
|
||||
REQUIRE(Clear(db).type == Error::Type::None);
|
||||
REQUIRE(Open(db, key).type == Error::Type::None);
|
||||
REQUIRE(Put(db, Key{ 0, 1 }, Test2()).type == Error::Type::None);
|
||||
REQUIRE(Put(db, Key{ 0, 1 }, Database::TaggedValue(Test1(), 1)).type
|
||||
== Error::Type::None);
|
||||
REQUIRE(PutIfEmpty(db, Key{ 0, 2 }, Test2()).type
|
||||
== Error::Type::None);
|
||||
REQUIRE(PutIfEmpty(db, Key{ 0, 2 }, Test1()).type
|
||||
== Error::Type::None);
|
||||
REQUIRE(CopyIfEmpty(db, Key{ 0, 1 }, Key{ 2, 0 }).type
|
||||
== Error::Type::None);
|
||||
REQUIRE(CopyIfEmpty(db, Key{ 0, 2 }, Key{ 2, 0 }).type
|
||||
== Error::Type::None);
|
||||
REQUIRE(Put(db, Key{ 0, 3 }, Test1()).type == Error::Type::None);
|
||||
REQUIRE(MoveIfEmpty(db, Key{ 0, 3 }, Key{ 3, 0 }).type
|
||||
== Error::Type::None);
|
||||
REQUIRE(MoveIfEmpty(db, Key{ 0, 2 }, Key{ 3, 0 }).type
|
||||
== Error::Type::None);
|
||||
Close(db);
|
||||
}
|
||||
SECTION("reading and writing db") {
|
||||
Database db(name, Settings);
|
||||
|
||||
REQUIRE(Open(db, key).type == Error::Type::None);
|
||||
REQUIRE((Get(db, Key{ 0, 1 }) == Test1()));
|
||||
const auto withTag1 = GetWithTag(db, Key{ 0, 1 });
|
||||
REQUIRE(((withTag1.bytes == Test1()) && (withTag1.tag == 1)));
|
||||
REQUIRE(Put(db, Key{ 1, 0 }, Test2()).type == Error::Type::None);
|
||||
const auto withTag2 = GetWithTag(db, Key{ 1, 0 });
|
||||
REQUIRE(((withTag2.bytes == Test2()) && (withTag2.tag == 0)));
|
||||
REQUIRE(Get(db, Key{ 1, 1 }).isEmpty());
|
||||
REQUIRE((Get(db, Key{ 0, 2 }) == Test2()));
|
||||
REQUIRE((Get(db, Key{ 2, 0 }) == Test1()));
|
||||
REQUIRE(Get(db, Key{ 0, 3 }).isEmpty());
|
||||
REQUIRE((Get(db, Key{ 3, 0 }) == Test1()));
|
||||
|
||||
REQUIRE(Put(db, Key{ 5, 1 }, Database::TaggedValue(Test1(), 1)).type
|
||||
== Error::Type::None);
|
||||
REQUIRE(Put(db, Key{ 6, 1 }, Database::TaggedValue(Test2(), 1)).type
|
||||
== Error::Type::None);
|
||||
REQUIRE(Put(db, Key{ 5, 2 }, Database::TaggedValue(Test1(), 2)).type
|
||||
== Error::Type::None);
|
||||
REQUIRE(Put(db, Key{ 6, 2 }, Database::TaggedValue(Test2(), 2)).type
|
||||
== Error::Type::None);
|
||||
REQUIRE(Put(db, Key{ 5, 3 }, Database::TaggedValue(Test1(), 3)).type
|
||||
== Error::Type::None);
|
||||
REQUIRE(Put(db, Key{ 6, 3 }, Database::TaggedValue(Test2(), 3)).type
|
||||
== Error::Type::None);
|
||||
Close(db);
|
||||
}
|
||||
SECTION("reading db") {
|
||||
Database db(name, Settings);
|
||||
|
||||
REQUIRE(Open(db, key).type == Error::Type::None);
|
||||
REQUIRE((Get(db, Key{ 0, 1 }) == Test1()));
|
||||
REQUIRE((Get(db, Key{ 1, 0 }) == Test2()));
|
||||
Close(db);
|
||||
}
|
||||
SECTION("deleting in db by tag") {
|
||||
Database db(name, Settings);
|
||||
|
||||
REQUIRE(Open(db, key).type == Error::Type::None);
|
||||
REQUIRE(ClearByTag(db, 2).type == Error::Type::None);
|
||||
REQUIRE((Get(db, Key{ 1, 0 }) == Test2()));
|
||||
|
||||
const auto withTag1 = GetWithTag(db, Key{ 5, 1 });
|
||||
REQUIRE(((withTag1.bytes == Test1()) && (withTag1.tag == 1)));
|
||||
const auto withTag2 = GetWithTag(db, Key{ 6, 1 });
|
||||
REQUIRE(((withTag2.bytes == Test2()) && (withTag2.tag == 1)));
|
||||
REQUIRE(Get(db, Key{ 5, 2 }).isEmpty());
|
||||
REQUIRE(Get(db, Key{ 6, 2 }).isEmpty());
|
||||
const auto withTag3 = GetWithTag(db, Key{ 5, 3 });
|
||||
REQUIRE(((withTag3.bytes == Test1()) && (withTag3.tag == 3)));
|
||||
const auto withTag4 = GetWithTag(db, Key{ 6, 3 });
|
||||
REQUIRE(((withTag4.bytes == Test2()) && (withTag4.tag == 3)));
|
||||
Close(db);
|
||||
}
|
||||
SECTION("overwriting values") {
|
||||
Database db(name, Settings);
|
||||
|
||||
REQUIRE(Open(db, key).type == Error::Type::None);
|
||||
const auto path = GetBinlogPath();
|
||||
REQUIRE((Get(db, Key{ 0, 1 }) == Test1()));
|
||||
const auto size = QFile(path).size();
|
||||
REQUIRE(Put(db, Key{ 0, 1 }, Test2()).type == Error::Type::None);
|
||||
const auto next = QFile(path).size();
|
||||
REQUIRE(next > size);
|
||||
REQUIRE((Get(db, Key{ 0, 1 }) == Test2()));
|
||||
REQUIRE(Put(db, Key{ 0, 1 }, Test2()).type == Error::Type::None);
|
||||
const auto same = QFile(path).size();
|
||||
REQUIRE(same == next);
|
||||
Close(db);
|
||||
}
|
||||
SECTION("reading db in many chunks") {
|
||||
auto settings = Settings;
|
||||
settings.readBlockSize = 512;
|
||||
settings.maxBundledRecords = 5;
|
||||
settings.trackEstimatedTime = true;
|
||||
Database db(name, settings);
|
||||
|
||||
const auto count = 30U;
|
||||
|
||||
REQUIRE(Clear(db).type == Error::Type::None);
|
||||
REQUIRE(Open(db, key).type == Error::Type::None);
|
||||
for (auto i = 0U; i != count; ++i) {
|
||||
auto value = Test1();
|
||||
value[0] = char('A') + i;
|
||||
const auto result = Put(db, Key{ i, i * 2 }, std::move(value));
|
||||
REQUIRE(result.type == Error::Type::None);
|
||||
}
|
||||
Close(db);
|
||||
|
||||
REQUIRE(Open(db, key).type == Error::Type::None);
|
||||
for (auto i = 0U; i != count; ++i) {
|
||||
auto value = Test1();
|
||||
value[0] = char('A') + i;
|
||||
REQUIRE((Get(db, Key{ i, i * 2 }) == value));
|
||||
}
|
||||
Close(db);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("cache db remove", "[storage_cache_database]") {
|
||||
if (!DisableLargeTest) {
|
||||
return;
|
||||
}
|
||||
SECTION("db remove deletes value") {
|
||||
Database db(name, Settings);
|
||||
|
||||
REQUIRE(Clear(db).type == Error::Type::None);
|
||||
REQUIRE(Open(db, key).type == Error::Type::None);
|
||||
REQUIRE(Put(db, Key{ 0, 1 }, Test1()).type == Error::Type::None);
|
||||
REQUIRE(Put(db, Key{ 1, 0 }, Test2()).type == Error::Type::None);
|
||||
Remove(db, Key{ 0, 1 });
|
||||
REQUIRE(Get(db, Key{ 0, 1 }).isEmpty());
|
||||
REQUIRE((Get(db, Key{ 1, 0 }) == Test2()));
|
||||
Close(db);
|
||||
}
|
||||
SECTION("db remove deletes value permanently") {
|
||||
Database db(name, Settings);
|
||||
|
||||
REQUIRE(Open(db, key).type == Error::Type::None);
|
||||
REQUIRE(Get(db, Key{ 0, 1 }).isEmpty());
|
||||
REQUIRE((Get(db, Key{ 1, 0 }) == Test2()));
|
||||
Close(db);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("cache db bundled actions", "[storage_cache_database]") {
|
||||
if (!DisableLargeTest) {
|
||||
return;
|
||||
}
|
||||
SECTION("db touched written lazily") {
|
||||
auto settings = Settings;
|
||||
settings.trackEstimatedTime = true;
|
||||
Database db(name, settings);
|
||||
|
||||
REQUIRE(Clear(db).type == Error::Type::None);
|
||||
REQUIRE(Open(db, key).type == Error::Type::None);
|
||||
const auto path = GetBinlogPath();
|
||||
REQUIRE(Put(db, Key{ 0, 1 }, Test1()).type == Error::Type::None);
|
||||
const auto size = QFile(path).size();
|
||||
REQUIRE((Get(db, Key{ 0, 1 }) == Test1()));
|
||||
REQUIRE(QFile(path).size() == size);
|
||||
AdvanceTime(2);
|
||||
Get(db, Key{ 0, 1 });
|
||||
REQUIRE(QFile(path).size() > size);
|
||||
Close(db);
|
||||
}
|
||||
SECTION("db touched written on close") {
|
||||
auto settings = Settings;
|
||||
settings.trackEstimatedTime = true;
|
||||
Database db(name, settings);
|
||||
|
||||
REQUIRE(Clear(db).type == Error::Type::None);
|
||||
REQUIRE(Open(db, key).type == Error::Type::None);
|
||||
const auto path = GetBinlogPath();
|
||||
REQUIRE(Put(db, Key{ 0, 1 }, Test1()).type == Error::Type::None);
|
||||
const auto size = QFile(path).size();
|
||||
REQUIRE((Get(db, Key{ 0, 1 }) == Test1()));
|
||||
REQUIRE(QFile(path).size() == size);
|
||||
Close(db);
|
||||
REQUIRE(QFile(path).size() > size);
|
||||
}
|
||||
SECTION("db remove written lazily") {
|
||||
Database db(name, Settings);
|
||||
|
||||
REQUIRE(Clear(db).type == Error::Type::None);
|
||||
REQUIRE(Open(db, key).type == Error::Type::None);
|
||||
const auto path = GetBinlogPath();
|
||||
REQUIRE(Put(db, Key{ 0, 1 }, Test1()).type == Error::Type::None);
|
||||
const auto size = QFile(path).size();
|
||||
Remove(db, Key{ 0, 1 });
|
||||
REQUIRE(QFile(path).size() == size);
|
||||
AdvanceTime(2);
|
||||
REQUIRE(QFile(path).size() > size);
|
||||
Close(db);
|
||||
}
|
||||
SECTION("db remove written on close") {
|
||||
Database db(name, Settings);
|
||||
|
||||
REQUIRE(Clear(db).type == Error::Type::None);
|
||||
REQUIRE(Open(db, key).type == Error::Type::None);
|
||||
const auto path = GetBinlogPath();
|
||||
REQUIRE(Put(db, Key{ 0, 1 }, Test1()).type == Error::Type::None);
|
||||
const auto size = QFile(path).size();
|
||||
Remove(db, Key{ 0, 1 });
|
||||
REQUIRE(QFile(path).size() == size);
|
||||
Close(db);
|
||||
REQUIRE(QFile(path).size() > size);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("cache db limits", "[storage_cache_database]") {
|
||||
if (DisableLimitsTests || !DisableLargeTest) {
|
||||
return;
|
||||
}
|
||||
SECTION("db both limit") {
|
||||
auto settings = Settings;
|
||||
settings.trackEstimatedTime = true;
|
||||
settings.totalSizeLimit = 17 * 3 + 1;
|
||||
settings.totalTimeLimit = 4;
|
||||
Database db(name, settings);
|
||||
|
||||
db.clear(nullptr);
|
||||
db.open(base::duplicate(key), nullptr);
|
||||
db.put(Key{ 0, 1 }, Test1(), nullptr);
|
||||
db.put(Key{ 1, 0 }, Test2(), nullptr);
|
||||
AdvanceTime(2);
|
||||
db.get(Key{ 1, 0 }, nullptr);
|
||||
AdvanceTime(3);
|
||||
db.put(Key{ 1, 1 }, Test1(), nullptr);
|
||||
db.put(Key{ 2, 0 }, Test2(), nullptr);
|
||||
db.put(Key{ 0, 2 }, Test1(), nullptr);
|
||||
AdvanceTime(2);
|
||||
REQUIRE(Get(db, Key{ 0, 1 }).isEmpty());
|
||||
REQUIRE(Get(db, Key{ 1, 0 }).isEmpty());
|
||||
REQUIRE((Get(db, Key{ 1, 1 }) == Test1()));
|
||||
REQUIRE((Get(db, Key{ 2, 0 }) == Test2()));
|
||||
REQUIRE((Get(db, Key{ 0, 2 }) == Test1()));
|
||||
Close(db);
|
||||
}
|
||||
SECTION("db size limit") {
|
||||
auto settings = Settings;
|
||||
settings.trackEstimatedTime = true;
|
||||
settings.totalSizeLimit = 17 * 3 + 1;
|
||||
Database db(name, settings);
|
||||
|
||||
db.clear(nullptr);
|
||||
db.open(base::duplicate(key), nullptr);
|
||||
db.put(Key{ 0, 1 }, Test1(), nullptr);
|
||||
AdvanceTime(2);
|
||||
db.put(Key{ 1, 0 }, Test2(), nullptr);
|
||||
AdvanceTime(2);
|
||||
db.put(Key{ 1, 1 }, Test1(), nullptr);
|
||||
db.get(Key{ 0, 1 }, nullptr);
|
||||
AdvanceTime(2);
|
||||
db.put(Key{ 2, 0 }, Test2(), nullptr);
|
||||
|
||||
// Removing { 1, 0 } will be scheduled.
|
||||
REQUIRE((Get(db, Key{ 0, 1 }) == Test1()));
|
||||
REQUIRE((Get(db, Key{ 1, 1 }) == Test1()));
|
||||
REQUIRE((Get(db, Key{ 2, 0 }) == Test2()));
|
||||
AdvanceTime(2);
|
||||
|
||||
// Removing { 1, 0 } performed.
|
||||
REQUIRE(Get(db, Key{ 1, 0 }).isEmpty());
|
||||
REQUIRE((Get(db, Key{ 1, 1 }) == Test1()));
|
||||
db.put(Key{ 0, 2 }, Test1(), nullptr);
|
||||
REQUIRE(Put(db, Key{ 2, 2 }, Test2()).type == Error::Type::None);
|
||||
|
||||
// Removing { 0, 1 } and { 2, 0 } will be scheduled.
|
||||
AdvanceTime(2);
|
||||
|
||||
// Removing { 0, 1 } and { 2, 0 } performed.
|
||||
REQUIRE(Get(db, Key{ 0, 1 }).isEmpty());
|
||||
REQUIRE(Get(db, Key{ 2, 0 }).isEmpty());
|
||||
REQUIRE((Get(db, Key{ 1, 1 }) == Test1()));
|
||||
REQUIRE((Get(db, Key{ 0, 2 }) == Test1()));
|
||||
REQUIRE((Get(db, Key{ 2, 2 }) == Test2()));
|
||||
Close(db);
|
||||
}
|
||||
SECTION("db time limit") {
|
||||
auto settings = Settings;
|
||||
settings.trackEstimatedTime = true;
|
||||
settings.totalTimeLimit = 3;
|
||||
Database db(name, settings);
|
||||
|
||||
db.clear(nullptr);
|
||||
db.open(base::duplicate(key), nullptr);
|
||||
db.put(Key{ 0, 1 }, Test1(), nullptr);
|
||||
db.put(Key{ 1, 0 }, Test2(), nullptr);
|
||||
db.put(Key{ 1, 1 }, Test1(), nullptr);
|
||||
db.put(Key{ 2, 0 }, Test2(), nullptr);
|
||||
AdvanceTime(1);
|
||||
db.get(Key{ 1, 0 }, nullptr);
|
||||
db.get(Key{ 1, 1 }, nullptr);
|
||||
AdvanceTime(1);
|
||||
db.get(Key{ 1, 0 }, nullptr);
|
||||
db.get(Key{ 0, 1 }, nullptr);
|
||||
AdvanceTime(1);
|
||||
db.get(Key{ 1, 0 }, nullptr);
|
||||
db.get(Key{ 0, 1 }, nullptr);
|
||||
AdvanceTime(3);
|
||||
REQUIRE(Get(db, Key{ 2, 0 }).isEmpty());
|
||||
REQUIRE(Get(db, Key{ 1, 1 }).isEmpty());
|
||||
REQUIRE((Get(db, Key{ 1, 0 }) == Test2()));
|
||||
REQUIRE((Get(db, Key{ 0, 1 }) == Test1()));
|
||||
Close(db);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("large db", "[storage_cache_database]") {
|
||||
if (DisableLargeTest) {
|
||||
return;
|
||||
}
|
||||
SECTION("time tracking large db") {
|
||||
auto settings = Database::Settings();
|
||||
settings.writeBundleDelay = crl::time(1000);
|
||||
settings.maxDataSize = 20;
|
||||
settings.totalSizeLimit = 1024 * 1024;
|
||||
settings.totalTimeLimit = 120;
|
||||
settings.pruneTimeout = crl::time(1500);
|
||||
settings.compactAfterExcess = 1024 * 1024;
|
||||
settings.trackEstimatedTime = true;
|
||||
Database db(name, settings);
|
||||
|
||||
//REQUIRE(Clear(db).type == Error::Type::None);
|
||||
REQUIRE(Open(db, key).type == Error::Type::None);
|
||||
|
||||
const auto key = [](int index) {
|
||||
return Key{ uint64(index) * 2, (uint64(index) << 32) + 3 };
|
||||
};
|
||||
const auto kWriteRecords = 100 * 1024;
|
||||
for (auto i = 0; i != kWriteRecords; ++i) {
|
||||
db.put(key(i), Test1(), nullptr);
|
||||
const auto j = i ? (rand() % i) : 0;
|
||||
if (i % 1024 == 1023) {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
||||
Get(db, key(j));
|
||||
} else {
|
||||
db.get(key(j), nullptr);
|
||||
}
|
||||
}
|
||||
|
||||
Close(db);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,138 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#include "storage/cache/storage_cache_types.h"
|
||||
|
||||
#include <QtCore/QDir>
|
||||
|
||||
namespace Storage {
|
||||
namespace Cache {
|
||||
namespace details {
|
||||
namespace {
|
||||
|
||||
template <typename Packed>
|
||||
inline Packed ReadTo(size_type count) {
|
||||
Expects(count >= 0 && count < (1 << (Packed().size() * 8)));
|
||||
|
||||
auto result = Packed();
|
||||
for (auto &element : result) {
|
||||
element = uint8(count & 0xFF);
|
||||
count >>= 8;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
template <typename Packed>
|
||||
inline size_type ReadFrom(const Packed &count) {
|
||||
auto result = size_type();
|
||||
for (auto &element : (count | ranges::view::reverse)) {
|
||||
result <<= 8;
|
||||
result |= size_type(element);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
template <typename Packed>
|
||||
inline size_type ValidateStrictCount(const Packed &count) {
|
||||
const auto result = ReadFrom(count);
|
||||
return (result != 0) ? result : -1;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TaggedValue::TaggedValue(QByteArray &&bytes, uint8 tag)
|
||||
: bytes(std::move(bytes)), tag(tag) {
|
||||
}
|
||||
|
||||
QString ComputeBasePath(const QString &original) {
|
||||
const auto result = QDir(original).absolutePath();
|
||||
return result.endsWith('/') ? result : (result + '/');
|
||||
}
|
||||
|
||||
QString VersionFilePath(const QString &base) {
|
||||
Expects(base.endsWith('/'));
|
||||
|
||||
return base + QStringLiteral("version");
|
||||
}
|
||||
|
||||
std::optional<Version> ReadVersionValue(const QString &base) {
|
||||
QFile file(VersionFilePath(base));
|
||||
if (!file.open(QIODevice::ReadOnly)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
const auto bytes = file.read(sizeof(Version));
|
||||
if (bytes.size() != sizeof(Version)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
return *reinterpret_cast<const Version*>(bytes.data());
|
||||
}
|
||||
|
||||
bool WriteVersionValue(const QString &base, Version value) {
|
||||
if (!QDir().mkpath(base)) {
|
||||
return false;
|
||||
}
|
||||
const auto bytes = QByteArray::fromRawData(
|
||||
reinterpret_cast<const char*>(&value),
|
||||
sizeof(value));
|
||||
QFile file(VersionFilePath(base));
|
||||
if (!file.open(QIODevice::WriteOnly)) {
|
||||
return false;
|
||||
} else if (file.write(bytes) != bytes.size()) {
|
||||
return false;
|
||||
}
|
||||
return file.flush();
|
||||
}
|
||||
|
||||
BasicHeader::BasicHeader()
|
||||
: format(static_cast<uint32>(Format::Format_0))
|
||||
, flags(0) {
|
||||
}
|
||||
|
||||
void Store::setSize(size_type size) {
|
||||
this->size = ReadTo<EntrySize>(size);
|
||||
}
|
||||
|
||||
size_type Store::getSize() const {
|
||||
return ReadFrom(size);
|
||||
}
|
||||
|
||||
MultiStore::MultiStore(size_type count)
|
||||
: type(kType)
|
||||
, count(ReadTo<RecordsCount>(count)) {
|
||||
Expects(count >= 0 && count < kBundledRecordsLimit);
|
||||
}
|
||||
|
||||
size_type MultiStore::validateCount() const {
|
||||
return ValidateStrictCount(count);
|
||||
}
|
||||
|
||||
MultiRemove::MultiRemove(size_type count)
|
||||
: type(kType)
|
||||
, count(ReadTo<RecordsCount>(count)) {
|
||||
Expects(count >= 0 && count < kBundledRecordsLimit);
|
||||
}
|
||||
|
||||
size_type MultiRemove::validateCount() const {
|
||||
return ValidateStrictCount(count);
|
||||
}
|
||||
|
||||
MultiAccess::MultiAccess(
|
||||
EstimatedTimePoint time,
|
||||
size_type count)
|
||||
: type(kType)
|
||||
, count(ReadTo<RecordsCount>(count))
|
||||
, time(time) {
|
||||
Expects(count >= 0 && count < kBundledRecordsLimit);
|
||||
}
|
||||
|
||||
size_type MultiAccess::validateCount() const {
|
||||
return ReadFrom(count);
|
||||
}
|
||||
|
||||
} // namespace details
|
||||
} // namespace Cache
|
||||
} // namespace Storage
|
||||
|
|
@ -1,239 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include "base/basic_types.h"
|
||||
#include "base/flat_map.h"
|
||||
#include "base/optional.h"
|
||||
#include <crl/crl_time.h>
|
||||
#include <QtCore/QString>
|
||||
#include <QtCore/QByteArray>
|
||||
|
||||
namespace Storage {
|
||||
namespace Cache {
|
||||
|
||||
struct Key {
|
||||
uint64 high = 0;
|
||||
uint64 low = 0;
|
||||
};
|
||||
|
||||
inline bool operator==(const Key &a, const Key &b) {
|
||||
return (a.high == b.high) && (a.low == b.low);
|
||||
}
|
||||
|
||||
inline bool operator!=(const Key &a, const Key &b) {
|
||||
return !(a == b);
|
||||
}
|
||||
|
||||
inline bool operator<(const Key &a, const Key &b) {
|
||||
return std::tie(a.high, a.low) < std::tie(b.high, b.low);
|
||||
}
|
||||
|
||||
struct Error {
|
||||
enum class Type {
|
||||
None,
|
||||
IO,
|
||||
WrongKey,
|
||||
LockFailed,
|
||||
};
|
||||
Type type = Type::None;
|
||||
QString path;
|
||||
|
||||
static Error NoError();
|
||||
};
|
||||
|
||||
inline Error Error::NoError() {
|
||||
return Error();
|
||||
}
|
||||
|
||||
namespace details {
|
||||
|
||||
using RecordType = uint8;
|
||||
using PlaceId = std::array<uint8, 7>;
|
||||
using EntrySize = std::array<uint8, 3>;
|
||||
using RecordsCount = std::array<uint8, 3>;
|
||||
|
||||
constexpr auto kRecordSizeUnknown = size_type(-1);
|
||||
constexpr auto kRecordSizeInvalid = size_type(-2);
|
||||
constexpr auto kBundledRecordsLimit
|
||||
= size_type(1 << (RecordsCount().size() * 8));
|
||||
constexpr auto kDataSizeLimit = size_type(1 << (EntrySize().size() * 8));
|
||||
|
||||
struct Settings {
|
||||
size_type maxBundledRecords = 16 * 1024;
|
||||
size_type readBlockSize = 8 * 1024 * 1024;
|
||||
size_type maxDataSize = (kDataSizeLimit - 1);
|
||||
crl::time writeBundleDelay = 15 * 60 * crl::time(1000);
|
||||
size_type staleRemoveChunk = 256;
|
||||
|
||||
int64 compactAfterExcess = 8 * 1024 * 1024;
|
||||
int64 compactAfterFullSize = 0;
|
||||
size_type compactChunkSize = 16 * 1024;
|
||||
|
||||
bool trackEstimatedTime = true;
|
||||
int64 totalSizeLimit = 1024 * 1024 * 1024;
|
||||
size_type totalTimeLimit = 31 * 24 * 60 * 60; // One month in seconds.
|
||||
crl::time pruneTimeout = 5 * crl::time(1000);
|
||||
crl::time maxPruneCheckTimeout = 3600 * crl::time(1000);
|
||||
|
||||
bool clearOnWrongKey = false;
|
||||
};
|
||||
|
||||
struct SettingsUpdate {
|
||||
int64 totalSizeLimit = Settings().totalSizeLimit;
|
||||
size_type totalTimeLimit = Settings().totalTimeLimit;
|
||||
};
|
||||
|
||||
struct TaggedValue {
|
||||
TaggedValue() = default;
|
||||
TaggedValue(QByteArray &&bytes, uint8 tag);
|
||||
|
||||
QByteArray bytes;
|
||||
uint8 tag = 0;
|
||||
};
|
||||
|
||||
struct TaggedSummary {
|
||||
size_type count = 0;
|
||||
int64 totalSize = 0;
|
||||
};
|
||||
struct Stats {
|
||||
TaggedSummary full;
|
||||
base::flat_map<uint8, TaggedSummary> tagged;
|
||||
bool clearing = false;
|
||||
};
|
||||
|
||||
using Version = int32;
|
||||
|
||||
QString ComputeBasePath(const QString &original);
|
||||
QString VersionFilePath(const QString &base);
|
||||
std::optional<Version> ReadVersionValue(const QString &base);
|
||||
bool WriteVersionValue(const QString &base, Version value);
|
||||
|
||||
template <typename Record>
|
||||
constexpr auto GoodForEncryption = ((sizeof(Record) & 0x0F) == 0);
|
||||
|
||||
enum class Format : uint32 {
|
||||
Format_0,
|
||||
};
|
||||
|
||||
struct BasicHeader {
|
||||
BasicHeader();
|
||||
|
||||
static constexpr auto kTrackEstimatedTime = 0x01U;
|
||||
|
||||
Format getFormat() const {
|
||||
return static_cast<Format>(format);
|
||||
}
|
||||
void setFormat(Format format) {
|
||||
this->format = static_cast<uint32>(format);
|
||||
}
|
||||
|
||||
uint32 format : 8;
|
||||
uint32 flags : 24;
|
||||
uint32 systemTime = 0;
|
||||
uint32 reserved1 = 0;
|
||||
uint32 reserved2 = 0;
|
||||
};
|
||||
|
||||
struct EstimatedTimePoint {
|
||||
uint32 relative1 = 0;
|
||||
uint32 relative2 = 0;
|
||||
uint32 system = 0;
|
||||
|
||||
void setRelative(uint64 value) {
|
||||
relative1 = uint32(value & 0xFFFFFFFFU);
|
||||
relative2 = uint32((value >> 32) & 0xFFFFFFFFU);
|
||||
}
|
||||
uint64 getRelative() const {
|
||||
return uint64(relative1) | (uint64(relative2) << 32);
|
||||
}
|
||||
};
|
||||
|
||||
struct Store {
|
||||
static constexpr auto kType = RecordType(0x01);
|
||||
|
||||
void setSize(size_type size);
|
||||
size_type getSize() const;
|
||||
|
||||
RecordType type = kType;
|
||||
uint8 tag = 0;
|
||||
EntrySize size = { { 0 } };
|
||||
PlaceId place = { { 0 } };
|
||||
uint32 checksum = 0;
|
||||
Key key;
|
||||
};
|
||||
|
||||
struct StoreWithTime : Store {
|
||||
EstimatedTimePoint time;
|
||||
uint32 reserved = 0;
|
||||
};
|
||||
|
||||
struct MultiStore {
|
||||
static constexpr auto kType = RecordType(0x02);
|
||||
|
||||
explicit MultiStore(size_type count = 0);
|
||||
|
||||
RecordType type = kType;
|
||||
RecordsCount count = { { 0 } };
|
||||
uint32 reserved1 = 0;
|
||||
uint32 reserved2 = 0;
|
||||
uint32 reserved3 = 0;
|
||||
|
||||
using Part = Store;
|
||||
size_type validateCount() const;
|
||||
};
|
||||
struct MultiStoreWithTime : MultiStore {
|
||||
using MultiStore::MultiStore;
|
||||
|
||||
using Part = StoreWithTime;
|
||||
};
|
||||
|
||||
struct MultiRemove {
|
||||
static constexpr auto kType = RecordType(0x03);
|
||||
|
||||
explicit MultiRemove(size_type count = 0);
|
||||
|
||||
RecordType type = kType;
|
||||
RecordsCount count = { { 0 } };
|
||||
uint32 reserved1 = 0;
|
||||
uint32 reserved2 = 0;
|
||||
uint32 reserved3 = 0;
|
||||
|
||||
using Part = Key;
|
||||
size_type validateCount() const;
|
||||
};
|
||||
|
||||
struct MultiAccess {
|
||||
static constexpr auto kType = RecordType(0x04);
|
||||
|
||||
explicit MultiAccess(
|
||||
EstimatedTimePoint time,
|
||||
size_type count = 0);
|
||||
|
||||
RecordType type = kType;
|
||||
RecordsCount count = { { 0 } };
|
||||
EstimatedTimePoint time;
|
||||
|
||||
using Part = Key;
|
||||
size_type validateCount() const;
|
||||
};
|
||||
|
||||
} // namespace details
|
||||
} // namespace Cache
|
||||
} // namespace Storage
|
||||
|
||||
namespace std {
|
||||
|
||||
template <>
|
||||
struct hash<Storage::Cache::Key> {
|
||||
size_t operator()(const Storage::Cache::Key &key) const {
|
||||
return (hash<uint64>()(key.high) ^ hash<uint64>()(key.low));
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace std
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#include "storage/storage_clear_legacy.h"
|
||||
|
||||
#include <crl/crl_async.h>
|
||||
|
||||
namespace Storage {
|
||||
namespace {
|
||||
|
||||
constexpr auto kClearPartSize = size_type(10000);
|
||||
|
||||
} // namespace
|
||||
|
||||
void ClearLegacyFilesPart(
|
||||
const QString &base,
|
||||
CollectGoodFiles filter,
|
||||
base::flat_set<QString> &&skip = {}) {
|
||||
filter([
|
||||
=,
|
||||
files = details::CollectFiles(base, kClearPartSize, skip)
|
||||
](base::flat_set<QString> &&skip) mutable {
|
||||
crl::async([
|
||||
=,
|
||||
files = std::move(files),
|
||||
skip = std::move(skip)
|
||||
]() mutable {
|
||||
for (const auto &name : files) {
|
||||
if (!skip.contains(name)
|
||||
&& !details::RemoveLegacyFile(base + name)) {
|
||||
skip.emplace(name);
|
||||
}
|
||||
}
|
||||
if (files.size() == kClearPartSize) {
|
||||
ClearLegacyFilesPart(base, filter, std::move(skip));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
void ClearLegacyFiles(const QString &base, CollectGoodFiles filter) {
|
||||
Expects(base.endsWith('/'));
|
||||
|
||||
crl::async([=] {
|
||||
ClearLegacyFilesPart(base, std::move(filter));
|
||||
});
|
||||
}
|
||||
|
||||
} // namespace Storage
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
namespace Storage {
|
||||
|
||||
using CollectGoodFiles = Fn<void(FnMut<void(base::flat_set<QString>&&)>)>;
|
||||
|
||||
void ClearLegacyFiles(const QString &base, CollectGoodFiles filter);
|
||||
|
||||
namespace details {
|
||||
|
||||
std::vector<QString> CollectFiles(
|
||||
const QString &base,
|
||||
size_type limit,
|
||||
const base::flat_set<QString> &skip);
|
||||
|
||||
bool RemoveLegacyFile(const QString &path);
|
||||
|
||||
} // namespace details
|
||||
} // namespace Storage
|
||||
|
|
@ -1,101 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#include "storage/storage_clear_legacy.h"
|
||||
|
||||
#include <sys/stat.h>
|
||||
#include <dirent.h>
|
||||
#include <unistd.h>
|
||||
|
||||
namespace Storage {
|
||||
namespace details {
|
||||
|
||||
std::vector<QString> CollectFiles(
|
||||
const QString &base,
|
||||
size_type limit,
|
||||
const base::flat_set<QString> &skip) {
|
||||
Expects(base.endsWith('/'));
|
||||
Expects(limit > 0);
|
||||
|
||||
const auto path = QFile::encodeName(base);
|
||||
const auto folder = path.mid(0, path.size() - 1);
|
||||
const auto directory = opendir(folder.constData());
|
||||
if (!directory) {
|
||||
return {};
|
||||
}
|
||||
const auto guard = gsl::finally([&] { closedir(directory); });
|
||||
|
||||
auto result = std::vector<QString>();
|
||||
while (const auto entry = readdir(directory)) {
|
||||
const auto local = entry->d_name;
|
||||
if (!strcmp(local, ".") || !strcmp(local, "..")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const auto full = path + QByteArray(local);
|
||||
const auto data = full.constData();
|
||||
struct stat statbuf = { 0 };
|
||||
if (stat(full.constData(), &statbuf) != 0 || S_ISDIR(statbuf.st_mode)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto name = QFile::decodeName(local);
|
||||
if (!skip.contains(name)) {
|
||||
result.push_back(std::move(name));
|
||||
}
|
||||
if (result.size() == limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
|
||||
// // It looks like POSIX solution works fine on macOS so no need for Cocoa solution.
|
||||
//
|
||||
// NSString *native = [NSString stringWithUTF8String:utf8.constData()];
|
||||
// NSFileManager *manager = [NSFileManager defaultManager];
|
||||
// NSArray *properties = [NSArray arrayWithObject:NSURLIsDirectoryKey];
|
||||
// NSDirectoryEnumerator *enumerator = [manager
|
||||
// enumeratorAtURL:[NSURL fileURLWithPath:native]
|
||||
// includingPropertiesForKeys:properties
|
||||
// options:0
|
||||
// errorHandler:^(NSURL *url, NSError *error) {
|
||||
// return NO;
|
||||
// }];
|
||||
//
|
||||
// auto result = std::vector<QString>();
|
||||
// for (NSURL *url in enumerator) {
|
||||
// NSNumber *isDirectory = nil;
|
||||
// NSError *error = nil;
|
||||
// if (![url getResourceValue:&isDirectory forKey:NSURLIsDirectoryKey error:&error]) {
|
||||
// break;
|
||||
// } else if ([isDirectory boolValue]) {
|
||||
// continue;
|
||||
// }
|
||||
// NSString *full = [url path];
|
||||
// NSRange r = [full rangeOfString:native];
|
||||
// if (r.location != 0) {
|
||||
// break;
|
||||
// }
|
||||
// NSString *file = [full substringFromIndex:r.length + 1];
|
||||
// auto name = QString::fromUtf8([file cStringUsingEncoding:NSUTF8StringEncoding]);
|
||||
// if (!skip.contains(name)) {
|
||||
// result.push_back(std::move(name));
|
||||
// }
|
||||
// if (result.size() == limit) {
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
// return result;
|
||||
}
|
||||
|
||||
bool RemoveLegacyFile(const QString &path) {
|
||||
const auto native = QFile::encodeName(path);
|
||||
return unlink(native.constData()) == 0;
|
||||
}
|
||||
|
||||
} // namespace details
|
||||
} // namespace Storage
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#include "storage/storage_clear_legacy.h"
|
||||
|
||||
#include "base/platform/win/base_windows_h.h"
|
||||
|
||||
namespace Storage {
|
||||
namespace details {
|
||||
|
||||
std::vector<QString> CollectFiles(
|
||||
const QString &base,
|
||||
size_type limit,
|
||||
const base::flat_set<QString> &skip) {
|
||||
Expects(base.endsWith('/'));
|
||||
Expects(limit > 0);
|
||||
|
||||
const auto native = QDir::toNativeSeparators(base).toStdWString();
|
||||
const auto search = native + L'*';
|
||||
|
||||
auto data = WIN32_FIND_DATA{ 0 };
|
||||
const auto handle = FindFirstFileEx(
|
||||
search.c_str(),
|
||||
FindExInfoBasic,
|
||||
&data,
|
||||
FindExSearchNameMatch,
|
||||
nullptr,
|
||||
0);
|
||||
if (handle == INVALID_HANDLE_VALUE) {
|
||||
return {};
|
||||
}
|
||||
const auto guard = gsl::finally([&] { FindClose(handle); });
|
||||
|
||||
auto result = std::vector<QString>();
|
||||
do {
|
||||
const auto full = native + data.cFileName;
|
||||
if (data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
|
||||
continue;
|
||||
}
|
||||
const auto file = QString::fromWCharArray(
|
||||
data.cFileName,
|
||||
full.size() - native.size());
|
||||
auto name = QDir::fromNativeSeparators(file);
|
||||
if (!skip.contains(name)) {
|
||||
result.push_back(std::move(name));
|
||||
}
|
||||
} while (result.size() != limit && FindNextFile(handle, &data));
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
bool RemoveLegacyFile(const QString &path) {
|
||||
const auto native = QDir::toNativeSeparators(path).toStdWString();
|
||||
return (::DeleteFile(native.c_str()) != 0);
|
||||
}
|
||||
|
||||
} // namespace details
|
||||
} // namespace Storage
|
||||
|
|
@ -1,104 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#include "storage/storage_databases.h"
|
||||
|
||||
#include "storage/cache/storage_cache_database.h"
|
||||
|
||||
namespace Storage {
|
||||
|
||||
DatabasePointer::DatabasePointer(
|
||||
not_null<Databases*> owner,
|
||||
const std::unique_ptr<Cache::Database> &value)
|
||||
: _value(value.get())
|
||||
, _owner(owner) {
|
||||
}
|
||||
|
||||
DatabasePointer::DatabasePointer(DatabasePointer &&other)
|
||||
: _value(base::take(other._value))
|
||||
, _owner(other._owner) {
|
||||
}
|
||||
|
||||
DatabasePointer &DatabasePointer::operator=(DatabasePointer &&other) {
|
||||
if (this != &other) {
|
||||
destroy();
|
||||
_owner = other._owner;
|
||||
_value = base::take(other._value);
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
DatabasePointer::~DatabasePointer() {
|
||||
destroy();
|
||||
}
|
||||
|
||||
Cache::Database *DatabasePointer::get() const {
|
||||
return _value;
|
||||
}
|
||||
|
||||
Cache::Database &DatabasePointer::operator*() const {
|
||||
Expects(_value != nullptr);
|
||||
|
||||
return *get();
|
||||
}
|
||||
|
||||
Cache::Database *DatabasePointer::operator->() const {
|
||||
Expects(_value != nullptr);
|
||||
|
||||
return get();
|
||||
}
|
||||
|
||||
DatabasePointer::operator bool() const {
|
||||
return get() != nullptr;
|
||||
}
|
||||
|
||||
void DatabasePointer::destroy() {
|
||||
if (const auto value = base::take(_value)) {
|
||||
_owner->destroy(value);
|
||||
}
|
||||
}
|
||||
|
||||
Databases::Kept::Kept(std::unique_ptr<Cache::Database> &&database)
|
||||
: database(std::move(database)) {
|
||||
}
|
||||
|
||||
DatabasePointer Databases::get(
|
||||
const QString &path,
|
||||
const Cache::details::Settings &settings) {
|
||||
if (const auto i = _map.find(path); i != end(_map)) {
|
||||
auto &kept = i->second;
|
||||
Assert(kept.destroying.alive());
|
||||
kept.destroying = nullptr;
|
||||
kept.database->reconfigure(settings);
|
||||
return DatabasePointer(this, kept.database);
|
||||
}
|
||||
const auto [i, ok] = _map.emplace(
|
||||
path,
|
||||
std::make_unique<Cache::Database>(path, settings));
|
||||
return DatabasePointer(this, i->second.database);
|
||||
}
|
||||
|
||||
void Databases::destroy(Cache::Database *database) {
|
||||
for (auto &entry : _map) {
|
||||
const auto &path = entry.first; // Need to capture it in lambda.
|
||||
auto &kept = entry.second;
|
||||
if (kept.database.get() == database) {
|
||||
Assert(!kept.destroying.alive());
|
||||
database->close();
|
||||
database->waitForCleaner([
|
||||
=,
|
||||
guard = kept.destroying.make_guard()
|
||||
]() mutable {
|
||||
crl::on_main(std::move(guard), [=] {
|
||||
_map.erase(path);
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Storage
|
||||
|
|
@ -1,71 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include "storage/cache/storage_cache_database.h"
|
||||
#include "base/binary_guard.h"
|
||||
|
||||
namespace Storage {
|
||||
namespace Cache {
|
||||
namespace details {
|
||||
struct Settings;
|
||||
} // namespace details
|
||||
class Database;
|
||||
} // namespace Cache
|
||||
|
||||
class Databases;
|
||||
|
||||
class DatabasePointer {
|
||||
public:
|
||||
DatabasePointer(const DatabasePointer &other) = delete;
|
||||
DatabasePointer(DatabasePointer &&other);
|
||||
DatabasePointer &operator=(const DatabasePointer &other) = delete;
|
||||
DatabasePointer &operator=(DatabasePointer &&other);
|
||||
~DatabasePointer();
|
||||
|
||||
Cache::Database *get() const;
|
||||
Cache::Database &operator*() const;
|
||||
Cache::Database *operator->() const;
|
||||
explicit operator bool() const;
|
||||
|
||||
private:
|
||||
friend class Databases;
|
||||
|
||||
DatabasePointer(
|
||||
not_null<Databases*> owner,
|
||||
const std::unique_ptr<Cache::Database> &value);
|
||||
void destroy();
|
||||
|
||||
Cache::Database *_value = nullptr;
|
||||
not_null<Databases*> _owner;
|
||||
|
||||
};
|
||||
|
||||
class Databases {
|
||||
public:
|
||||
DatabasePointer get(
|
||||
const QString &path,
|
||||
const Cache::details::Settings &settings);
|
||||
|
||||
private:
|
||||
friend class DatabasePointer;
|
||||
|
||||
struct Kept {
|
||||
Kept(std::unique_ptr<Cache::Database> &&database);
|
||||
|
||||
std::unique_ptr<Cache::Database> database;
|
||||
base::binary_guard destroying;
|
||||
};
|
||||
|
||||
void destroy(Cache::Database *database);
|
||||
|
||||
std::map<QString, Kept> _map;
|
||||
|
||||
};
|
||||
|
||||
} // namespace Storage
|
||||
|
|
@ -1,348 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#include "storage/storage_encrypted_file.h"
|
||||
|
||||
#include "base/openssl_help.h"
|
||||
|
||||
namespace Storage {
|
||||
namespace {
|
||||
|
||||
constexpr auto kBlockSize = CtrState::kBlockSize;
|
||||
|
||||
enum class Format : uint32 {
|
||||
Format_0,
|
||||
};
|
||||
|
||||
struct BasicHeader {
|
||||
BasicHeader();
|
||||
|
||||
void setFormat(Format format) {
|
||||
this->format = static_cast<uint32>(format);
|
||||
}
|
||||
Format getFormat() const {
|
||||
return static_cast<Format>(format);
|
||||
}
|
||||
|
||||
bytes::array<kSaltSize> salt = { { bytes::type() } };
|
||||
uint32 format : 8;
|
||||
uint32 reserved1 : 24;
|
||||
uint32 reserved2 = 0;
|
||||
uint64 applicationVersion = 0;
|
||||
bytes::array<openssl::kSha256Size> checksum = { { bytes::type() } };
|
||||
};
|
||||
|
||||
BasicHeader::BasicHeader()
|
||||
: format(static_cast<uint32>(Format::Format_0))
|
||||
, reserved1(0) {
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
File::Result File::open(
|
||||
const QString &path,
|
||||
Mode mode,
|
||||
const EncryptionKey &key) {
|
||||
close();
|
||||
|
||||
const auto info = QFileInfo(path);
|
||||
const auto dir = info.absoluteDir();
|
||||
if (mode != Mode::Read && !dir.exists()) {
|
||||
if (!QDir().mkpath(dir.absolutePath())) {
|
||||
return Result::Failed;
|
||||
}
|
||||
}
|
||||
|
||||
_data.setFileName(info.absoluteFilePath());
|
||||
const auto result = attemptOpen(mode, key);
|
||||
if (result != Result::Success) {
|
||||
close();
|
||||
}
|
||||
return result;
|
||||
|
||||
static_assert(sizeof(BasicHeader) == kSaltSize
|
||||
+ sizeof(uint64) * 2
|
||||
+ openssl::kSha256Size, "Unexpected paddings in the header.");
|
||||
static_assert(
|
||||
(sizeof(BasicHeader) - kSaltSize) % kBlockSize == 0,
|
||||
"Not way to encrypt the header.");
|
||||
}
|
||||
|
||||
File::Result File::attemptOpen(Mode mode, const EncryptionKey &key) {
|
||||
switch (mode) {
|
||||
case Mode::Read: return attemptOpenForRead(key);
|
||||
case Mode::ReadAppend: return attemptOpenForReadAppend(key);
|
||||
case Mode::Write: return attemptOpenForWrite(key);
|
||||
}
|
||||
Unexpected("Mode in Storage::File::attemptOpen.");
|
||||
}
|
||||
|
||||
File::Result File::attemptOpenForRead(const EncryptionKey &key) {
|
||||
if (!_data.open(QIODevice::ReadOnly)) {
|
||||
return Result::Failed;
|
||||
}
|
||||
return readHeader(key);
|
||||
}
|
||||
|
||||
File::Result File::attemptOpenForReadAppend(const EncryptionKey &key) {
|
||||
if (!_lock.lock(_data, QIODevice::ReadWrite)) {
|
||||
return Result::LockFailed;
|
||||
}
|
||||
const auto size = _data.size();
|
||||
if (!size) {
|
||||
return writeHeader(key) ? Result::Success : Result::Failed;
|
||||
}
|
||||
return readHeader(key);
|
||||
}
|
||||
|
||||
File::Result File::attemptOpenForWrite(const EncryptionKey &key) {
|
||||
if (!_lock.lock(_data, QIODevice::WriteOnly)) {
|
||||
return Result::LockFailed;
|
||||
}
|
||||
return writeHeader(key) ? Result::Success : Result::Failed;
|
||||
}
|
||||
|
||||
bool File::writeHeader(const EncryptionKey &key) {
|
||||
Expects(!_state.has_value());
|
||||
Expects(_data.pos() == 0);
|
||||
|
||||
const auto magic = bytes::make_span("TDEF");
|
||||
if (!writePlain(magic.subspan(0, FileLock::kSkipBytes))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
auto header = BasicHeader();
|
||||
bytes::set_random(header.salt);
|
||||
_state = key.prepareCtrState(header.salt);
|
||||
|
||||
const auto headerBytes = bytes::object_as_span(&header);
|
||||
const auto checkSize = headerBytes.size() - header.checksum.size();
|
||||
bytes::copy(
|
||||
header.checksum,
|
||||
openssl::Sha256(
|
||||
key.data(),
|
||||
headerBytes.subspan(0, checkSize)));
|
||||
|
||||
if (writePlain(header.salt) != header.salt.size()) {
|
||||
return false;
|
||||
} else if (!write(headerBytes.subspan(header.salt.size()))) {
|
||||
return false;
|
||||
}
|
||||
_dataSize = 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
File::Result File::readHeader(const EncryptionKey &key) {
|
||||
Expects(!_state.has_value());
|
||||
Expects(_data.pos() == 0);
|
||||
|
||||
if (!_data.seek(FileLock::kSkipBytes)) {
|
||||
return Result::Failed;
|
||||
}
|
||||
auto header = BasicHeader();
|
||||
const auto headerBytes = bytes::object_as_span(&header);
|
||||
if (readPlain(headerBytes) != headerBytes.size()) {
|
||||
return Result::Failed;
|
||||
}
|
||||
_state = key.prepareCtrState(header.salt);
|
||||
decrypt(headerBytes.subspan(header.salt.size()));
|
||||
|
||||
const auto checkSize = headerBytes.size() - header.checksum.size();
|
||||
const auto checksum = openssl::Sha256(
|
||||
key.data(),
|
||||
headerBytes.subspan(0, checkSize));
|
||||
if (bytes::compare(header.checksum, checksum) != 0) {
|
||||
return Result::WrongKey;
|
||||
} else if (header.getFormat() != Format::Format_0) {
|
||||
return Result::Failed;
|
||||
}
|
||||
_dataSize = _data.size()
|
||||
- int64(sizeof(BasicHeader))
|
||||
- FileLock::kSkipBytes;
|
||||
Assert(_dataSize >= 0);
|
||||
if (const auto bad = (_dataSize % kBlockSize)) {
|
||||
_dataSize -= bad;
|
||||
}
|
||||
return Result::Success;
|
||||
}
|
||||
|
||||
size_type File::readPlain(bytes::span bytes) {
|
||||
return _data.read(reinterpret_cast<char*>(bytes.data()), bytes.size());
|
||||
}
|
||||
|
||||
size_type File::writePlain(bytes::const_span bytes) {
|
||||
return _data.write(
|
||||
reinterpret_cast<const char*>(bytes.data()),
|
||||
bytes.size());
|
||||
}
|
||||
|
||||
void File::decrypt(bytes::span bytes) {
|
||||
Expects(_state.has_value());
|
||||
|
||||
_state->decrypt(bytes, _encryptionOffset);
|
||||
_encryptionOffset += bytes.size();
|
||||
}
|
||||
|
||||
void File::encrypt(bytes::span bytes) {
|
||||
Expects(_state.has_value());
|
||||
|
||||
_state->encrypt(bytes, _encryptionOffset);
|
||||
_encryptionOffset += bytes.size();
|
||||
}
|
||||
|
||||
size_type File::read(bytes::span bytes) {
|
||||
Expects(bytes.size() % kBlockSize == 0);
|
||||
|
||||
auto count = readPlain(bytes);
|
||||
if (const auto back = -(count % kBlockSize)) {
|
||||
if (!_data.seek(_data.pos() + back)) {
|
||||
return 0;
|
||||
}
|
||||
count += back;
|
||||
}
|
||||
if (count) {
|
||||
decrypt(bytes.subspan(0, count));
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
bool File::write(bytes::span bytes) {
|
||||
Expects(bytes.size() % kBlockSize == 0);
|
||||
|
||||
if (!isOpen()) {
|
||||
return false;
|
||||
}
|
||||
encrypt(bytes);
|
||||
const auto count = writePlain(bytes);
|
||||
if (count == bytes.size()) {
|
||||
_dataSize = std::max(_dataSize, offset());
|
||||
} else {
|
||||
decryptBack(bytes);
|
||||
if (count > 0) {
|
||||
_data.seek(_data.pos() - count);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void File::decryptBack(bytes::span bytes) {
|
||||
Expects(_encryptionOffset >= bytes.size());
|
||||
|
||||
_encryptionOffset -= bytes.size();
|
||||
decrypt(bytes);
|
||||
_encryptionOffset -= bytes.size();
|
||||
}
|
||||
|
||||
size_type File::readWithPadding(bytes::span bytes) {
|
||||
const auto size = bytes.size();
|
||||
const auto part = size % kBlockSize;
|
||||
const auto good = size - part;
|
||||
if (good) {
|
||||
const auto succeed = read(bytes.subspan(0, good));
|
||||
if (succeed != good) {
|
||||
return succeed;
|
||||
}
|
||||
}
|
||||
if (!part) {
|
||||
return good;
|
||||
}
|
||||
auto storage = bytes::array<kBlockSize>();
|
||||
const auto padded = bytes::make_span(storage);
|
||||
const auto succeed = read(padded);
|
||||
if (!succeed) {
|
||||
return good;
|
||||
}
|
||||
Assert(succeed == kBlockSize);
|
||||
bytes::copy(bytes.subspan(good), padded.subspan(0, part));
|
||||
return size;
|
||||
}
|
||||
|
||||
bool File::writeWithPadding(bytes::span bytes) {
|
||||
const auto size = bytes.size();
|
||||
const auto part = size % kBlockSize;
|
||||
const auto good = size - part;
|
||||
if (good && !write(bytes.subspan(0, good))) {
|
||||
return false;
|
||||
}
|
||||
if (!part) {
|
||||
return true;
|
||||
}
|
||||
auto storage = bytes::array<kBlockSize>();
|
||||
const auto padded = bytes::make_span(storage);
|
||||
bytes::copy(padded, bytes.subspan(good));
|
||||
bytes::set_random(padded.subspan(part));
|
||||
if (write(padded)) {
|
||||
return true;
|
||||
}
|
||||
if (good) {
|
||||
decryptBack(bytes.subspan(0, good));
|
||||
_data.seek(_data.pos() - good);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool File::flush() {
|
||||
return _data.flush();
|
||||
}
|
||||
|
||||
void File::close() {
|
||||
_lock.unlock();
|
||||
_data.close();
|
||||
_data.setFileName(QString());
|
||||
_dataSize = _encryptionOffset = 0;
|
||||
_state = std::nullopt;
|
||||
}
|
||||
|
||||
bool File::isOpen() const {
|
||||
return _data.isOpen();
|
||||
}
|
||||
|
||||
int64 File::size() const {
|
||||
return _dataSize;
|
||||
}
|
||||
|
||||
int64 File::offset() const {
|
||||
const auto realOffset = kSaltSize + _encryptionOffset;
|
||||
const auto skipOffset = sizeof(BasicHeader);
|
||||
return (realOffset >= skipOffset) ? (realOffset - skipOffset) : 0;
|
||||
}
|
||||
|
||||
bool File::seek(int64 offset) {
|
||||
const auto realOffset = sizeof(BasicHeader) + offset;
|
||||
if (offset < 0 || offset > _dataSize) {
|
||||
return false;
|
||||
} else if (!_data.seek(FileLock::kSkipBytes + realOffset)) {
|
||||
return false;
|
||||
}
|
||||
_encryptionOffset = realOffset - kSaltSize;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool File::Move(const QString &from, const QString &to) {
|
||||
QFile source(from);
|
||||
if (!source.exists()) {
|
||||
return false;
|
||||
}
|
||||
QFile destination(to);
|
||||
if (destination.exists()) {
|
||||
{
|
||||
FileLock locker;
|
||||
if (!locker.lock(destination, QIODevice::WriteOnly)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
destination.close();
|
||||
if (!destination.remove()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return source.rename(to);
|
||||
}
|
||||
|
||||
|
||||
} // namespace Storage
|
||||
|
|
@ -1,73 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include "storage/storage_file_lock.h"
|
||||
#include "storage/storage_encryption.h"
|
||||
#include "base/bytes.h"
|
||||
#include "base/optional.h"
|
||||
|
||||
namespace Storage {
|
||||
|
||||
class File {
|
||||
public:
|
||||
enum class Mode {
|
||||
Read,
|
||||
ReadAppend,
|
||||
Write,
|
||||
};
|
||||
enum class Result {
|
||||
Failed,
|
||||
LockFailed,
|
||||
WrongKey,
|
||||
Success,
|
||||
};
|
||||
Result open(const QString &path, Mode mode, const EncryptionKey &key);
|
||||
|
||||
size_type read(bytes::span bytes);
|
||||
bool write(bytes::span bytes);
|
||||
|
||||
size_type readWithPadding(bytes::span bytes);
|
||||
bool writeWithPadding(bytes::span bytes);
|
||||
|
||||
bool flush();
|
||||
|
||||
bool isOpen() const;
|
||||
int64 size() const;
|
||||
int64 offset() const;
|
||||
bool seek(int64 offset);
|
||||
|
||||
void close();
|
||||
|
||||
static bool Move(const QString &from, const QString &to);
|
||||
|
||||
private:
|
||||
Result attemptOpen(Mode mode, const EncryptionKey &key);
|
||||
Result attemptOpenForRead(const EncryptionKey &key);
|
||||
Result attemptOpenForReadAppend(const EncryptionKey &key);
|
||||
Result attemptOpenForWrite(const EncryptionKey &key);
|
||||
|
||||
bool writeHeader(const EncryptionKey &key);
|
||||
Result readHeader(const EncryptionKey &key);
|
||||
|
||||
size_type readPlain(bytes::span bytes);
|
||||
size_type writePlain(bytes::const_span bytes);
|
||||
void decrypt(bytes::span bytes);
|
||||
void encrypt(bytes::span bytes);
|
||||
void decryptBack(bytes::span bytes);
|
||||
|
||||
QFile _data;
|
||||
FileLock _lock;
|
||||
int64 _encryptionOffset = 0;
|
||||
int64 _dataSize = 0;
|
||||
|
||||
std::optional<CtrState> _state;
|
||||
|
||||
};
|
||||
|
||||
} // namespace Storage
|
||||
|
|
@ -1,247 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#include "catch.hpp"
|
||||
|
||||
#include "storage/storage_encrypted_file.h"
|
||||
|
||||
#include <QtCore/QThread>
|
||||
#include <QtCore/QCoreApplication>
|
||||
|
||||
#ifdef Q_OS_WIN
|
||||
#include "platform/win/windows_dlls.h"
|
||||
#endif // Q_OS_WIN
|
||||
|
||||
#include <QtCore/QProcess>
|
||||
|
||||
#include <thread>
|
||||
#ifdef Q_OS_MAC
|
||||
#include <mach-o/dyld.h>
|
||||
#elif defined Q_OS_LINUX // Q_OS_MAC
|
||||
#include <unistd.h>
|
||||
#endif // Q_OS_MAC || Q_OS_LINUX
|
||||
|
||||
extern int (*TestForkedMethod)();
|
||||
|
||||
const auto Key = Storage::EncryptionKey(bytes::make_vector(
|
||||
bytes::make_span("\
|
||||
abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\
|
||||
abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\
|
||||
abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\
|
||||
abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\
|
||||
").subspan(0, Storage::EncryptionKey::kSize)));
|
||||
|
||||
const auto Name = QString("test.file");
|
||||
|
||||
const auto Test1 = bytes::make_span("testbytetestbyte").subspan(0, 16);
|
||||
const auto Test2 = bytes::make_span("bytetestbytetest").subspan(0, 16);
|
||||
|
||||
struct ForkInit {
|
||||
static int Method() {
|
||||
Storage::File file;
|
||||
const auto result = file.open(
|
||||
Name,
|
||||
Storage::File::Mode::ReadAppend,
|
||||
Key);
|
||||
if (result != Storage::File::Result::Success) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
auto data = bytes::vector(16);
|
||||
const auto read = file.read(data);
|
||||
if (read != data.size()) {
|
||||
return -1;
|
||||
} else if (data != bytes::make_vector(Test1)) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (!file.write(data) || !file.flush()) {
|
||||
return -1;
|
||||
}
|
||||
#ifdef _DEBUG
|
||||
while (true) {
|
||||
std::this_thread::sleep_for(std::chrono::seconds(1));
|
||||
}
|
||||
#else // _DEBUG
|
||||
std::this_thread::sleep_for(std::chrono::seconds(1));
|
||||
return 0;
|
||||
#endif // _DEBUG
|
||||
}
|
||||
ForkInit() {
|
||||
#ifdef Q_OS_WIN
|
||||
Platform::Dlls::start();
|
||||
#endif // Q_OS_WIN
|
||||
|
||||
TestForkedMethod = &ForkInit::Method;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
ForkInit ForkInitializer;
|
||||
QProcess ForkProcess;
|
||||
|
||||
TEST_CASE("simple encrypted file", "[storage_encrypted_file]") {
|
||||
SECTION("writing file") {
|
||||
Storage::File file;
|
||||
const auto result = file.open(
|
||||
Name,
|
||||
Storage::File::Mode::Write,
|
||||
Key);
|
||||
REQUIRE(result == Storage::File::Result::Success);
|
||||
|
||||
auto data = bytes::make_vector(Test1);
|
||||
const auto success = file.write(data);
|
||||
REQUIRE(success);
|
||||
}
|
||||
SECTION("reading and writing file") {
|
||||
Storage::File file;
|
||||
const auto result = file.open(
|
||||
Name,
|
||||
Storage::File::Mode::ReadAppend,
|
||||
Key);
|
||||
REQUIRE(result == Storage::File::Result::Success);
|
||||
|
||||
auto data = bytes::vector(Test1.size());
|
||||
const auto read = file.read(data);
|
||||
REQUIRE(read == data.size());
|
||||
REQUIRE(data == bytes::make_vector(Test1));
|
||||
|
||||
data = bytes::make_vector(Test2);
|
||||
const auto success = file.write(data);
|
||||
REQUIRE(success);
|
||||
}
|
||||
SECTION("offset and seek") {
|
||||
Storage::File file;
|
||||
const auto result = file.open(
|
||||
Name,
|
||||
Storage::File::Mode::ReadAppend,
|
||||
Key);
|
||||
REQUIRE(result == Storage::File::Result::Success);
|
||||
REQUIRE(file.offset() == 0);
|
||||
REQUIRE(file.size() == Test1.size() + Test2.size());
|
||||
|
||||
const auto success1 = file.seek(Test1.size());
|
||||
REQUIRE(success1);
|
||||
REQUIRE(file.offset() == Test1.size());
|
||||
|
||||
auto data = bytes::vector(Test2.size());
|
||||
const auto read = file.read(data);
|
||||
REQUIRE(read == data.size());
|
||||
REQUIRE(data == bytes::make_vector(Test2));
|
||||
REQUIRE(file.offset() == Test1.size() + Test2.size());
|
||||
REQUIRE(file.size() == Test1.size() + Test2.size());
|
||||
|
||||
const auto success2 = file.seek(Test1.size());
|
||||
REQUIRE(success2);
|
||||
REQUIRE(file.offset() == Test1.size());
|
||||
|
||||
data = bytes::make_vector(Test1);
|
||||
const auto success3 = file.write(data) && file.write(data);
|
||||
REQUIRE(success3);
|
||||
|
||||
REQUIRE(file.offset() == 3 * Test1.size());
|
||||
REQUIRE(file.size() == 3 * Test1.size());
|
||||
}
|
||||
SECTION("reading file") {
|
||||
Storage::File file;
|
||||
|
||||
const auto result = file.open(
|
||||
Name,
|
||||
Storage::File::Mode::Read,
|
||||
Key);
|
||||
REQUIRE(result == Storage::File::Result::Success);
|
||||
|
||||
auto data = bytes::vector(32);
|
||||
const auto read = file.read(data);
|
||||
REQUIRE(read == data.size());
|
||||
REQUIRE(data == bytes::concatenate(Test1, Test1));
|
||||
}
|
||||
SECTION("moving file") {
|
||||
const auto result = Storage::File::Move(Name, "other.file");
|
||||
REQUIRE(result);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("two process encrypted file", "[storage_encrypted_file]") {
|
||||
SECTION("writing file") {
|
||||
Storage::File file;
|
||||
const auto result = file.open(
|
||||
Name,
|
||||
Storage::File::Mode::Write,
|
||||
Key);
|
||||
REQUIRE(result == Storage::File::Result::Success);
|
||||
|
||||
auto data = bytes::make_vector(Test1);
|
||||
const auto success = file.write(data);
|
||||
REQUIRE(success);
|
||||
}
|
||||
SECTION("access from subprocess") {
|
||||
SECTION("start subprocess") {
|
||||
const auto application = []() -> QString {
|
||||
#ifdef Q_OS_WIN
|
||||
return "tests_storage.exe";
|
||||
#else // Q_OS_WIN
|
||||
constexpr auto kMaxPath = 1024;
|
||||
char result[kMaxPath] = { 0 };
|
||||
uint32_t size = kMaxPath;
|
||||
#ifdef Q_OS_MAC
|
||||
if (_NSGetExecutablePath(result, &size) == 0) {
|
||||
return result;
|
||||
}
|
||||
#else // Q_OS_MAC
|
||||
auto count = readlink("/proc/self/exe", result, size);
|
||||
if (count > 0) {
|
||||
return result;
|
||||
}
|
||||
#endif // Q_OS_MAC
|
||||
return "tests_storage";
|
||||
#endif // Q_OS_WIN
|
||||
}();
|
||||
|
||||
ForkProcess.start(application + " --forked");
|
||||
const auto started = ForkProcess.waitForStarted();
|
||||
REQUIRE(started);
|
||||
}
|
||||
SECTION("read subprocess result") {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(500));
|
||||
|
||||
Storage::File file;
|
||||
|
||||
const auto result = file.open(
|
||||
Name,
|
||||
Storage::File::Mode::Read,
|
||||
Key);
|
||||
REQUIRE(result == Storage::File::Result::Success);
|
||||
|
||||
auto data = bytes::vector(32);
|
||||
const auto read = file.read(data);
|
||||
REQUIRE(read == data.size());
|
||||
REQUIRE(data == bytes::concatenate(Test1, Test1));
|
||||
}
|
||||
SECTION("take subprocess result") {
|
||||
REQUIRE(ForkProcess.state() == QProcess::Running);
|
||||
|
||||
Storage::File file;
|
||||
|
||||
const auto result = file.open(
|
||||
Name,
|
||||
Storage::File::Mode::ReadAppend,
|
||||
Key);
|
||||
REQUIRE(result == Storage::File::Result::Success);
|
||||
|
||||
auto data = bytes::vector(32);
|
||||
const auto read = file.read(data);
|
||||
REQUIRE(read == data.size());
|
||||
REQUIRE(data == bytes::concatenate(Test1, Test1));
|
||||
|
||||
const auto finished = ForkProcess.waitForFinished(0);
|
||||
REQUIRE(finished);
|
||||
REQUIRE(ForkProcess.state() == QProcess::NotRunning);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,109 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#include "storage/storage_encryption.h"
|
||||
|
||||
#include "base/openssl_help.h"
|
||||
|
||||
namespace Storage {
|
||||
|
||||
CtrState::CtrState(bytes::const_span key, bytes::const_span iv) {
|
||||
Expects(key.size() == _key.size());
|
||||
Expects(iv.size() == _iv.size());
|
||||
|
||||
bytes::copy(_key, key);
|
||||
bytes::copy(_iv, iv);
|
||||
}
|
||||
|
||||
template <typename Method>
|
||||
void CtrState::process(bytes::span data, int64 offset, Method method) {
|
||||
Expects((data.size() % kBlockSize) == 0);
|
||||
Expects((offset % kBlockSize) == 0);
|
||||
|
||||
AES_KEY aes;
|
||||
AES_set_encrypt_key(
|
||||
reinterpret_cast<const uchar*>(_key.data()),
|
||||
_key.size() * CHAR_BIT,
|
||||
&aes);
|
||||
|
||||
unsigned char ecountBuf[kBlockSize] = { 0 };
|
||||
unsigned int offsetInBlock = 0;
|
||||
const auto blockIndex = offset / kBlockSize;
|
||||
auto iv = incrementedIv(blockIndex);
|
||||
|
||||
CRYPTO_ctr128_encrypt(
|
||||
reinterpret_cast<const uchar*>(data.data()),
|
||||
reinterpret_cast<uchar*>(data.data()),
|
||||
data.size(),
|
||||
&aes,
|
||||
reinterpret_cast<unsigned char*>(iv.data()),
|
||||
ecountBuf,
|
||||
&offsetInBlock,
|
||||
(block128_f)method);
|
||||
}
|
||||
|
||||
auto CtrState::incrementedIv(int64 blockIndex)
|
||||
-> bytes::array<kIvSize> {
|
||||
Expects(blockIndex >= 0);
|
||||
|
||||
if (!blockIndex) {
|
||||
return _iv;
|
||||
}
|
||||
auto result = _iv;
|
||||
auto digits = kIvSize;
|
||||
auto increment = uint64(blockIndex);
|
||||
do {
|
||||
--digits;
|
||||
increment += static_cast<uint64>(result[digits]);
|
||||
result[digits] = static_cast<bytes::type>(increment & 0xFFULL);
|
||||
increment >>= 8;
|
||||
} while (digits != 0 && increment != 0);
|
||||
return result;
|
||||
}
|
||||
|
||||
void CtrState::encrypt(bytes::span data, int64 offset) {
|
||||
return process(data, offset, AES_encrypt);
|
||||
}
|
||||
|
||||
void CtrState::decrypt(bytes::span data, int64 offset) {
|
||||
return process(data, offset, AES_encrypt);
|
||||
}
|
||||
|
||||
EncryptionKey::EncryptionKey(bytes::vector &&data)
|
||||
: _data(std::move(data)) {
|
||||
Expects(_data.size() == kSize);
|
||||
}
|
||||
|
||||
bool EncryptionKey::empty() const {
|
||||
return _data.empty();
|
||||
}
|
||||
|
||||
EncryptionKey::operator bool() const {
|
||||
return !empty();
|
||||
}
|
||||
|
||||
const bytes::vector &EncryptionKey::data() const {
|
||||
return _data;
|
||||
}
|
||||
|
||||
CtrState EncryptionKey::prepareCtrState(bytes::const_span salt) const {
|
||||
Expects(salt.size() == kSaltSize);
|
||||
|
||||
const auto data = bytes::make_span(_data);
|
||||
const auto key = openssl::Sha256(
|
||||
data.subspan(0, kSize / 2),
|
||||
salt.subspan(0, kSaltSize / 2));
|
||||
const auto iv = openssl::Sha256(
|
||||
data.subspan(kSize / 2),
|
||||
salt.subspan(kSaltSize / 2));
|
||||
|
||||
return CtrState(
|
||||
key,
|
||||
bytes::make_span(iv).subspan(0, CtrState::kIvSize));
|
||||
}
|
||||
|
||||
} // namespace Storage
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include "base/bytes.h"
|
||||
|
||||
namespace Storage {
|
||||
|
||||
constexpr auto kSaltSize = size_type(64);
|
||||
|
||||
class CtrState {
|
||||
public:
|
||||
static constexpr auto kBlockSize = size_type(16);
|
||||
static constexpr auto kKeySize = size_type(32);
|
||||
static constexpr auto kIvSize = kBlockSize;
|
||||
|
||||
CtrState(bytes::const_span key, bytes::const_span iv);
|
||||
|
||||
void encrypt(bytes::span data, int64 offset);
|
||||
void decrypt(bytes::span data, int64 offset);
|
||||
|
||||
private:
|
||||
template <typename Method>
|
||||
void process(bytes::span data, int64 offset, Method method);
|
||||
|
||||
bytes::array<kIvSize> incrementedIv(int64 blockIndex);
|
||||
|
||||
static constexpr auto EcountSize = kBlockSize;
|
||||
|
||||
bytes::array<kKeySize> _key;
|
||||
bytes::array<kIvSize> _iv;
|
||||
|
||||
};
|
||||
|
||||
class EncryptionKey {
|
||||
public:
|
||||
static constexpr auto kSize = size_type(256);
|
||||
|
||||
EncryptionKey() = default;
|
||||
explicit EncryptionKey(bytes::vector &&data);
|
||||
|
||||
bool empty() const;
|
||||
explicit operator bool() const;
|
||||
|
||||
const bytes::vector &data() const;
|
||||
CtrState prepareCtrState(bytes::const_span salt) const;
|
||||
|
||||
private:
|
||||
bytes::vector _data;
|
||||
|
||||
};
|
||||
|
||||
} // namespace Storage
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#include "storage/storage_pch.h"
|
||||
|
||||
// Precompiled header helper.
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
/*
|
||||
This file is part of Telegram Desktop,
|
||||
the official desktop application for the Telegram messaging service.
|
||||
|
||||
For license and copyright information please follow this link:
|
||||
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
|
||||
#include <QtCore/QFile>
|
||||
#include <QtCore/QFileInfo>
|
||||
#include <QtCore/QDir>
|
||||
#include <QtCore/QByteArray>
|
||||
#include <QtCore/QString>
|
||||
|
||||
#include <crl/crl.h>
|
||||
#include <rpl/rpl.h>
|
||||
|
||||
#include <vector>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
#include <set>
|
||||
|
||||
#include <range/v3/all.hpp>
|
||||
|
||||
#include "base/flat_map.h"
|
||||
#include "base/flat_set.h"
|
||||
#include "base/optional.h"
|
||||
#include "base/openssl_help.h"
|
||||
|
||||
#include "logs.h"
|
||||
Loading…
Reference in New Issue