Write date/time inside Store cache record.

Telegram/SourceFiles/storage/cache/storage_cache_database.cpp

@@ -100,7 +100,7 @@ QString PlaceFromId(PlaceId place) {
 }
 
 int32 GetUnixtime() {
-	return int32(time(nullptr));
+	return std::max(int32(time(nullptr)), 1);
 }
 
 enum class Format : uint32 {
@@ -110,9 +110,11 @@ enum class Format : uint32 {
 struct BasicHeader {
 	BasicHeader();
 
+	static constexpr auto kTrackEstimatedTime = 0x01U;
+
 	Format format : 8;
 	uint32 flags : 24;
-	uint32 date = 0;
+	uint32 systemTime = 0;
 	uint32 reserved1 = 0;
 	uint32 reserved2 = 0;
 };
@@ -123,6 +125,11 @@ BasicHeader::BasicHeader()
 , flags(0) {
 }
 
+struct EstimatedTimePoint {
+	uint32 system = 0;
+	uint32 relativeAdvancement = 0;
+};
+
 struct Store {
 	static constexpr auto kType = RecordType(0x01);
 
@@ -135,6 +142,13 @@ struct Store {
 };
 static_assert(GoodForEncryption<Store>);
 
+struct StoreWithTime : Store {
+	EstimatedTimePoint time;
+	uint32 reserved1 = 0;
+	uint32 reserved2 = 0;
+};
+static_assert(GoodForEncryption<StoreWithTime>);
+
 struct MultiStoreHeader {
 	static constexpr auto kType = RecordType(0x02);
 
@@ -146,16 +160,9 @@ struct MultiStoreHeader {
 	uint32 reserved2 = 0;
 	uint32 reserved3 = 0;
 };
-struct MultiStorePart {
+using MultiStorePart = Store;
-	uint8 reserved = 0;
+using MultiStoreWithTimePart = StoreWithTime;
-	uint8 tag = 0;
-	EntrySize size = { { 0 } };
-	PlaceId place = { { 0 } };
-	uint32 checksum = 0;
-	Key key;
-};
 static_assert(GoodForEncryption<MultiStoreHeader>);
-static_assert(GoodForEncryption<MultiStorePart>);
 
 MultiStoreHeader::MultiStoreHeader(size_type count)
 : type(kType)
@@ -186,34 +193,30 @@ MultiRemoveHeader::MultiRemoveHeader(size_type count)
 	Expects(count >= 0 && count < kBundledRecordsLimit);
 }
 
-struct MultiTouchedHeader {
+struct MultiAccessHeader {
 	static constexpr auto kType = RecordType(0x04);
 
-	MultiTouchedHeader(
+	explicit MultiAccessHeader(
-		uint32 time,
+		EstimatedTimePoint time,
-		uint32 advancement,
 		size_type count = 0);
 
 	RecordType type = kType;
 	RecordsCount count = { { 0 } };
-	uint32 timeAdvancement = 0;
+	EstimatedTimePoint time;
-	uint32 systemTime = 0;
 	uint32 reserved = 0;
 };
-struct MultiTouchedPart {
+struct MultiAccessPart {
 	Key key;
 };
-static_assert(GoodForEncryption<MultiTouchedHeader>);
+static_assert(GoodForEncryption<MultiAccessHeader>);
-static_assert(GoodForEncryption<MultiTouchedPart>);
+static_assert(GoodForEncryption<MultiAccessPart>);
 
-MultiTouchedHeader::MultiTouchedHeader(
+MultiAccessHeader::MultiAccessHeader(
-	uint32 time,
+	EstimatedTimePoint time,
-	uint32 advancement,
 	size_type count)
 : type(kType)
 , count(ReadTo<RecordsCount>(count))
-, timeAdvancement(advancement)
+, time(time) {
-, systemTime(time) {
 	Expects(count >= 0 && count < kBundledRecordsLimit);
 }
 
@@ -280,14 +283,28 @@ private:
 	size_type readBinlogRecords(bytes::const_span data);
 	size_type readBinlogRecordSize(bytes::const_span data) const;
 	bool readBinlogRecord(bytes::const_span data);
+	template <typename RecordStore>
+	bool readRecordStoreGeneric(bytes::const_span data);
 	bool readRecordStore(bytes::const_span data);
+	template <typename StorePart>
+	bool readRecordMultiStoreGeneric(bytes::const_span data);
 	bool readRecordMultiStore(bytes::const_span data);
 	bool readRecordMultiRemove(bytes::const_span data);
-	bool readRecordMultiTouched(bytes::const_span data);
+	bool readRecordMultiAccess(bytes::const_span data);
+	template <typename RecordStore, typename Postprocess>
+	bool processRecordStoreGeneric(
+		const RecordStore *record,
+		Postprocess &&postprocess);
+	bool processRecordStore(const Store *record, std::is_class<Store>);
+	bool processRecordStore(
+		const StoreWithTime *record,
+		std::is_class<StoreWithTime>);
 
 	void adjustRelativeTime();
 	void startDelayedPruning();
 	int64 countRelativeTime() const;
+	EstimatedTimePoint countTimePoint() const;
+	void applyTimePoint(EstimatedTimePoint time);
 	int64 pruneBeforeTime() const;
 	void prune();
 	void collectTimePrune(
@@ -307,12 +324,18 @@ private:
 
 	QString placePath(PlaceId place) const;
 	bool isFreePlace(PlaceId place) const;
+	template <typename StoreRecord>
+	QString writeKeyPlaceGeneric(
+		StoreRecord &&record,
+		const Key &key,
+		size_type size,
+		uint32 checksum);
 	QString writeKeyPlace(const Key &key, size_type size, uint32 checksum);
 	void writeMultiRemoveLazy();
 	void writeMultiRemove();
-	void writeMultiTouchedLazy();
+	void writeMultiAccessLazy();
-	void writeMultiTouched();
+	void writeMultiAccess();
-	void writeMultiTouchedBlock();
+	void writeMultiAccessBlock();
 	void writeBundlesLazy();
 	void writeBundles();
 
@@ -321,12 +344,12 @@ private:
 
 	crl::weak_on_queue<Database> _weak;
 	QString _base, _path;
-	Settings _settings;
+	const Settings _settings;
 	EncryptionKey _key;
 	File _binlog;
 	Map _map;
 	std::set<Key> _removing;
-	std::set<Key> _touched;
+	std::set<Key> _accessed;
 
 	int64 _relativeTime = 0;
 	int64 _timeCorrection = 0;
@@ -461,15 +484,21 @@ bool Database::readHeader() {
 		return false;
 	} else if (header.format != Format::Format_0) {
 		return false;
+	} else if (_settings.trackEstimatedTime
+		!= !!(header.flags & header.kTrackEstimatedTime)) {
+		return false;
 	}
-	_relativeTime = _latestSystemTime = header.date;
+	_relativeTime = _latestSystemTime = header.systemTime;
 	return true;
 }
 
 bool Database::writeHeader() {
 	auto header = BasicHeader();
-	const auto now = std::max(GetUnixtime(), 1);
+	const auto now = _settings.trackEstimatedTime ? GetUnixtime() : 0;
-	_relativeTime = _latestSystemTime = header.date = now;
+	_relativeTime = _latestSystemTime = header.systemTime = now;
+	if (_settings.trackEstimatedTime) {
+		header.flags |= header.kTrackEstimatedTime;
+	}
 	return _binlog.write(bytes::object_as_span(&header));
 }
 
@@ -502,7 +531,7 @@ void Database::readBinlog() {
 }
 
 int64 Database::countRelativeTime() const {
-	const auto now = std::max(GetUnixtime(), 1);
+	const auto now = GetUnixtime();
 	const auto delta = std::max(int64(now) - int64(_latestSystemTime), 0LL);
 	return _relativeTime + delta;
 }
@@ -514,7 +543,7 @@ int64 Database::pruneBeforeTime() const {
 }
 
 void Database::startDelayedPruning() {
-	if (_map.empty()) {
+	if (!_settings.trackEstimatedTime || _map.empty()) {
 		return;
 	}
 	const auto pruning = [&] {
@@ -630,9 +659,12 @@ void Database::collectSizePrune(
 }
 
 void Database::adjustRelativeTime() {
-	const auto now = std::max(GetUnixtime(), 1);
+	if (!_settings.trackEstimatedTime) {
+		return;
+	}
+	const auto now = GetUnixtime();
 	if (now < _latestSystemTime) {
-		writeMultiTouchedBlock();
+		writeMultiAccessBlock();
 	}
 }
 
@@ -658,16 +690,20 @@ size_type Database::readBinlogRecordSize(bytes::const_span data) const {
 
 	switch (static_cast<RecordType>(data[0])) {
 	case Store::kType:
-		return sizeof(Store);
+		return _settings.trackEstimatedTime
+			? sizeof(StoreWithTime)
+			: sizeof(Store);
 
 	case MultiStoreHeader::kType:
 		if (data.size() >= sizeof(MultiStoreHeader)) {
 			const auto header = reinterpret_cast<const MultiStoreHeader*>(
 				data.data());
 			const auto count = ReadFrom(header->count);
+			const auto size = _settings.trackEstimatedTime
+				? sizeof(MultiStoreWithTimePart)
+				: sizeof(MultiStorePart);
 			return (count > 0 && count < _settings.maxBundledRecords)
-				? (sizeof(MultiStoreHeader)
+				? (sizeof(MultiStoreHeader) + count * size)
-					+ count * sizeof(MultiStorePart))
 				: kRecordSizeInvalid;
 		}
 		return kRecordSizeUnknown;
@@ -684,14 +720,16 @@ size_type Database::readBinlogRecordSize(bytes::const_span data) const {
 		}
 		return kRecordSizeUnknown;
 
-	case MultiTouchedHeader::kType:
+	case MultiAccessHeader::kType:
-		if (data.size() >= sizeof(MultiTouchedHeader)) {
+		if (!_settings.trackEstimatedTime) {
-			const auto header = reinterpret_cast<const MultiTouchedHeader*>(
+			return kRecordSizeInvalid;
+		} else if (data.size() >= sizeof(MultiAccessHeader)) {
+			const auto header = reinterpret_cast<const MultiAccessHeader*>(
 				data.data());
 			const auto count = ReadFrom(header->count);
-			return (count > 0 && count < _settings.maxBundledRecords)
+			return (count >= 0 && count < _settings.maxBundledRecords)
-				? (sizeof(MultiTouchedHeader)
+				? (sizeof(MultiAccessHeader)
-					+ count * sizeof(MultiTouchedPart))
+					+ count * sizeof(MultiAccessPart))
 				: kRecordSizeInvalid;
 		}
 		return kRecordSizeUnknown;
@@ -713,61 +751,98 @@ bool Database::readBinlogRecord(bytes::const_span data) {
 	case MultiRemoveHeader::kType:
 		return readRecordMultiRemove(data);
 
-	case MultiTouchedHeader::kType:
+	case MultiAccessHeader::kType:
-		return readRecordMultiTouched(data);
+		return readRecordMultiAccess(data);
 
 	}
 	Unexpected("Bad type in Database::readBinlogRecord.");
 }
 
-bool Database::readRecordStore(bytes::const_span data) {
+template <typename RecordStore>
-	Expects(data.size() >= sizeof(Store));
+bool Database::readRecordStoreGeneric(bytes::const_span data) {
+	Expects(data.size() >= sizeof(RecordStore));
 
-	const auto record = reinterpret_cast<const Store*>(data.data());
+	return processRecordStore(
+		reinterpret_cast<const RecordStore*>(data.data()),
+		std::is_class<RecordStore>{});
+}
+
+template <typename RecordStore, typename Postprocess>
+bool Database::processRecordStoreGeneric(
+		const RecordStore *record,
+		Postprocess &&postprocess) {
 	const auto size = ReadFrom(record->size);
-	if (size > _settings.maxDataSize) {
+	if (size <= 0 || size > _settings.maxDataSize) {
 		return false;
 	}
-	setMapEntry(
+	auto entry = Entry(
-		record->key,
-		Entry(
 		record->place,
 		record->tag,
 		record->checksum,
 		size,
-			_relativeTime));
+		_relativeTime);
+	if (!postprocess(entry, record)) {
+		return false;
+	}
+	setMapEntry(record->key, std::move(entry));
 	return true;
 }
 
-bool Database::readRecordMultiStore(bytes::const_span data) {
+bool Database::processRecordStore(
+		const Store *record,
+		std::is_class<Store>) {
+	const auto postprocess = [](auto&&...) { return true; };
+	return processRecordStoreGeneric(record, postprocess);
+}
+
+bool Database::processRecordStore(
+		const StoreWithTime *record,
+		std::is_class<StoreWithTime>) {
+	const auto postprocess = [&](
+			Entry &entry,
+			not_null<const StoreWithTime*> record) {
+		applyTimePoint(record->time);
+		entry.useTime = _relativeTime;
+		return true;
+	};
+	return processRecordStoreGeneric(record, postprocess);
+}
+
+bool Database::readRecordStore(bytes::const_span data) {
+	if (!_settings.trackEstimatedTime) {
+		return readRecordStoreGeneric<Store>(data);
+	}
+	return readRecordStoreGeneric<StoreWithTime>(data);
+}
+
+template <typename StorePart>
+bool Database::readRecordMultiStoreGeneric(bytes::const_span data) {
 	Expects(data.size() >= sizeof(MultiStoreHeader));
 
 	const auto bytes = data.data();
 	const auto record = reinterpret_cast<const MultiStoreHeader*>(bytes);
 	const auto count = ReadFrom(record->count);
 	Assert(data.size() >= sizeof(MultiStoreHeader)
-		+ count * sizeof(MultiStorePart));
+		+ count * sizeof(StorePart));
 	const auto parts = gsl::make_span(
-		reinterpret_cast<const MultiStorePart*>(
+		reinterpret_cast<const StorePart*>(
 			bytes + sizeof(MultiStoreHeader)),
 		count);
 	for (const auto &part : parts) {
-		const auto size = ReadFrom(part.size);
+		if (!processRecordStore(&part, std::is_class<StorePart>{})) {
-		if (size > _settings.maxDataSize) {
 			return false;
 		}
-		setMapEntry(
-			part.key,
-			Entry(
-				part.place,
-				part.tag,
-				part.checksum,
-				size,
-				_relativeTime));
 	}
 	return true;
 }
 
+bool Database::readRecordMultiStore(bytes::const_span data) {
+	if (!_settings.trackEstimatedTime) {
+		return readRecordMultiStoreGeneric<MultiStorePart>(data);
+	}
+	return readRecordMultiStoreGeneric<MultiStoreWithTimePart>(data);
+}
+
 void Database::setMapEntry(const Key &key, Entry &&entry) {
 	auto &already = _map[key];
 	_totalSize += entry.size - already.size;
@@ -817,22 +892,38 @@ bool Database::readRecordMultiRemove(bytes::const_span data) {
 	return true;
 }
 
-bool Database::readRecordMultiTouched(bytes::const_span data) {
+EstimatedTimePoint Database::countTimePoint() const {
-	Expects(data.size() >= sizeof(MultiTouchedHeader));
+	const auto now = std::max(GetUnixtime(), 1);
+	const auto delta = std::max(int64(now) - int64(_latestSystemTime), 0LL);
+	auto result = EstimatedTimePoint();
+	result.system = now;
+	result.relativeAdvancement = std::min(
+		delta,
+		int64(_settings.maxTimeAdvancement));
+	return result;
+}
+
+void Database::applyTimePoint(EstimatedTimePoint time) {
+	_relativeTime += time.relativeAdvancement;
+	_latestSystemTime = time.system;
+}
+
+bool Database::readRecordMultiAccess(bytes::const_span data) {
+	Expects(data.size() >= sizeof(MultiAccessHeader));
+	Expects(_settings.trackEstimatedTime);
 
 	const auto bytes = data.data();
-	const auto record = reinterpret_cast<const MultiTouchedHeader*>(bytes);
+	const auto record = reinterpret_cast<const MultiAccessHeader*>(bytes);
-	if (record->timeAdvancement > _settings.maxTimeAdvancement) {
+	if (record->time.relativeAdvancement > _settings.maxTimeAdvancement) {
 		return false;
 	}
-	_relativeTime += record->timeAdvancement;
+	applyTimePoint(record->time);
-	_latestSystemTime = record->systemTime;
 	const auto count = ReadFrom(record->count);
-	Assert(data.size() >= sizeof(MultiTouchedHeader)
+	Assert(data.size() >= sizeof(MultiAccessHeader)
-		+ count * sizeof(MultiTouchedPart));
+		+ count * sizeof(MultiAccessPart));
 	const auto parts = gsl::make_span(
-		reinterpret_cast<const MultiTouchedPart*>(
+		reinterpret_cast<const MultiAccessPart*>(
-			bytes + sizeof(MultiTouchedHeader)),
+			bytes + sizeof(MultiAccessHeader)),
 		count);
 	for (const auto &part : parts) {
 		if (const auto i = _map.find(part.key); i != end(_map)) {
@@ -853,6 +944,12 @@ void Database::put(
 		const Key &key,
 		QByteArray value,
 		FnMut<void(Error)> done) {
+	if (value.isEmpty()) {
+		remove(key, [done = std::move(done)]() mutable {
+			done(Error::NoError());
+		});
+		return;
+	}
 	_removing.erase(key);
 
 	const auto checksum = CountChecksum(bytes::make_span(value));
@@ -881,9 +978,6 @@ void Database::put(
 		} else {
 			data.flush();
 			invokeCallback(done, Error::NoError());
-
-			_touched.emplace(key);
-			writeMultiTouchedLazy();
 			startDelayedPruning();
 		}
 	} break;
@@ -892,13 +986,14 @@ void Database::put(
 	}
 }
 
-QString Database::writeKeyPlace(
+template <typename StoreRecord>
+QString Database::writeKeyPlaceGeneric(
+		StoreRecord &&record,
 		const Key &key,
 		size_type size,
 		uint32 checksum) {
 	Expects(size <= _settings.maxDataSize);
 
-	auto record = Store();
 	record.key = key;
 	record.size = ReadTo<EntrySize>(size);
 	record.checksum = checksum;
@@ -920,6 +1015,25 @@ QString Database::writeKeyPlace(
 	return result;
 }
 
+QString Database::writeKeyPlace(
+		const Key &key,
+		size_type size,
+		uint32 checksum) {
+	if (!_settings.trackEstimatedTime) {
+		return writeKeyPlaceGeneric(Store(), key, size, checksum);
+	}
+	auto record = StoreWithTime();
+	record.time = countTimePoint();
+	if (record.time.relativeAdvancement * crl::time_type(1000)
+		< _settings.writeBundleDelay) {
+		// We don't want to produce a lot of unique relativeTime values.
+		// So if change in it is not large we stick to the old value.
+		record.time.system = _latestSystemTime;
+		record.time.relativeAdvancement = 0;
+	}
+	return writeKeyPlaceGeneric(std::move(record), key, size, checksum);
+}
+
 void Database::get(const Key &key, FnMut<void(QByteArray)> done) {
 	if (_removing.find(key) != end(_removing)) {
 		invokeCallback(done, QByteArray());
@@ -952,9 +1066,10 @@ void Database::get(const Key &key, FnMut<void(QByteArray)> done) {
 			invokeCallback(done, QByteArray());
 		} else {
 			invokeCallback(done, std::move(result));
-
+			if (_settings.trackEstimatedTime) {
-			_touched.emplace(key);
+				_accessed.emplace(key);
-			writeMultiTouchedLazy();
+				writeMultiAccessLazy();
+			}
 			startDelayedPruning();
 		}
 	} break;
@@ -1009,46 +1124,40 @@ void Database::writeMultiRemove() {
 	}
 }
 
-void Database::writeMultiTouchedLazy() {
+void Database::writeMultiAccessLazy() {
-	if (_touched.size() == _settings.maxBundledRecords) {
+	if (_accessed.size() == _settings.maxBundledRecords) {
-		writeMultiTouched();
+		writeMultiAccess();
 	} else {
 		writeBundlesLazy();
 	}
 }
 
-void Database::writeMultiTouched() {
+void Database::writeMultiAccess() {
-	if (!_touched.empty()) {
+	if (!_accessed.empty()) {
-		writeMultiTouchedBlock();
+		writeMultiAccessBlock();
 	}
 }
 
-void Database::writeMultiTouchedBlock() {
+void Database::writeMultiAccessBlock() {
-	Expects(_touched.size() <= _settings.maxBundledRecords);
+	Expects(_settings.trackEstimatedTime);
+	Expects(_accessed.size() <= _settings.maxBundledRecords);
 
-	const auto now = std::max(GetUnixtime(), 1);
+	const auto time = countTimePoint();
-	const auto delta = std::max(int64(now) - int64(_latestSystemTime), 0LL);
+	const auto size = _accessed.size();
-	const auto advancement = std::min(
+	auto header = MultiAccessHeader(time, size);
-		delta,
+	auto list = std::vector<MultiAccessPart>();
-		int64(_settings.maxTimeAdvancement));
-	const auto size = _touched.size();
-	auto header = MultiTouchedHeader(now, advancement, size);
-	auto list = std::vector<MultiTouchedPart>();
 	if (size > 0) {
 		list.reserve(size);
-		for (const auto &key : base::take(_touched)) {
+		for (const auto &key : base::take(_accessed)) {
 			list.push_back({ key });
 		}
 	}
-	_latestSystemTime = now;
+	applyTimePoint(time);
-	if (advancement > 0) {
-		_relativeTime += advancement;
 	for (const auto &entry : list) {
 		if (const auto i = _map.find(entry.key); i != end(_map)) {
 			i->second.useTime = _relativeTime;
 		}
 	}
-	}
 
 	if (_binlog.write(bytes::object_as_span(&header))) {
 		if (size > 0) {
@@ -1060,7 +1169,9 @@ void Database::writeMultiTouchedBlock() {
 
 void Database::writeBundles() {
 	writeMultiRemove();
-	writeMultiTouched();
+	if (_settings.trackEstimatedTime) {
+		writeMultiAccess();
+	}
 }
 
 void Database::createCleaner() {

Telegram/SourceFiles/storage/cache/storage_cache_database.h

@@ -40,15 +40,17 @@ inline bool operator<(const Key &a, const Key &b) {
 class Database {
 public:
 	struct Settings {
-		int64 totalSizeLimit = 1024 * 1024 * 1024;
-		size_type totalTimeLimit = 30 * 86400; // One month in seconds.
 		size_type maxBundledRecords = 16 * 1024;
 		size_type readBlockSize = 8 * 1024 * 1024;
 		size_type maxDataSize = 10 * 1024 * 1024;
 		crl::time_type writeBundleDelay = 15 * 60 * crl::time_type(1000);
+
+		bool trackEstimatedTime = true;
+		int64 totalSizeLimit = 1024 * 1024 * 1024;
+		size_type totalTimeLimit = 30 * 86400; // One month in seconds.
 		size_type maxTimeAdvancement = 365 * 86400; // One year in seconds.
 		crl::time_type pruneTimeout = 5 * crl::time_type(1000);
-		crl::time_type maxPruneCheckTimeout = 60 * 60 * crl::time_type(1000);
+		crl::time_type maxPruneCheckTimeout = 3600 * crl::time_type(1000);
 	};
 	Database(const QString &path, const Settings &settings);
 

Telegram/SourceFiles/storage/cache/storage_cache_database_tests.cpp

@@ -460,7 +460,7 @@ TEST_CASE("cache db limits", "[storage_cache_database]") {
 		AdvanceTime(1);
 		db.get(Key{ 1, 0 }, nullptr);
 		db.get(Key{ 0, 1 }, nullptr);
-		AdvanceTime(2);
+		AdvanceTime(3);
 		db.get(Key{ 2, 0 }, GetValue);
 		Semaphore.acquire();
 		REQUIRE(Value.isEmpty());