Use lz4 compression for cached frames.

Telegram/SourceFiles/lottie/lottie_cache.cpp

@@ -12,6 +12,8 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 #include "base/bytes.h"
 
 #include <QDataStream>
+#include <lz4.h>
+#include <range/v3/numeric/accumulate.hpp>
 
 namespace Lottie {
 namespace {
@@ -24,17 +26,32 @@ bool UncompressToRaw(AlignedStorage &to, bytes::const_span from) {
 	} else if (from.size() == to.rawSize()) {
 		memcpy(to.raw(), from.data(), from.size());
 		return true;
-	} else {
-		// #TODO stickers
-		return false;
 	}
+	const auto result = LZ4_decompress_safe(
+		reinterpret_cast<const char*>(from.data()),
+		static_cast<char*>(to.raw()),
+		from.size(),
+		to.rawSize());
+	return (result == to.rawSize());
 }
 
 void CompressFromRaw(QByteArray &to, const AlignedStorage &from) {
-	const auto size = to.size();
+	const auto size = from.rawSize();
-	to.resize(size + from.rawSize());
+	const auto max = LZ4_compressBound(size);
-	memcpy(to.data() + size, from.raw(), from.rawSize());
+	to.reserve(max);
-	// #TODO stickers
+	to.resize(max);
+	const auto compressed = LZ4_compress_default(
+		static_cast<const char*>(from.raw()),
+		to.data(),
+		size,
+		to.size());
+	Assert(compressed > 0);
+	if (compressed >= size) {
+		to.resize(size);
+		memcpy(to.data(), from.raw(), size);
+	} else {
+		to.resize(compressed);
+	}
 }
 
 void Decode(QImage &to, AlignedStorage &from, const QSize &fromSize) {
@@ -249,7 +266,6 @@ bool CacheState::readHeader(const FrameRequest &request) {
 		|| request.size(original) != size) {
 		return false;
 	}
-	_headerSize = stream.device()->pos();
 	_size = size;
 	_original = original;
 	_frameRate = frameRate;
@@ -276,7 +292,7 @@ bool CacheState::renderFrame(
 	} else if (request.size(_original) != _size) {
 		return false;
 	} else if (index == 0) {
-		_offset = _headerSize;
+		_offset = headerSize();
 		_offsetFrameIndex = 0;
 	}
 	if (!readCompressedDelta()) {
@@ -306,10 +322,8 @@ void CacheState::appendFrame(
 	}
 	if (index == 0) {
 		_size = request.size(_original);
-		writeHeader();
+		_compressedFrames.reserve(_framesCount);
 		prepareBuffers();
-	} else {
-		incrementFramesReady();
 	}
 	Encode(_uncompressed, frame, _size);
 	if (index == 0) {
@@ -320,43 +334,60 @@ void CacheState::appendFrame(
 		Xor(_uncompressed, _previous);
 		writeCompressedDelta();
 	}
+	if (++_framesReady == _framesCount) {
+		finalizeEncoding();
+	}
+}
+
+void CacheState::finalizeEncoding() {
+	const auto size = (_data.isEmpty() ? headerSize() : _data.size())
+		+ (_compressedFrames.size() * sizeof(qint32))
+		+ ranges::accumulate(
+			_compressedFrames,
+			0,
+			std::plus(),
+			&QByteArray::size);
+	if (_data.isEmpty()) {
+		_data.reserve(size);
+		writeHeader();
+	}
+	const auto offset = _data.size();
+	_data.resize(size);
+	auto to = _data.data() + offset;
+	for (const auto &block : _compressedFrames) {
+		const auto amount = qint32(block.size());
+		memcpy(to, &amount, sizeof(qint32));
+		to += sizeof(qint32);
+		memcpy(to, block.data(), amount);
+		to += amount;
+	}
+	_compressedFrames.clear();
+	_compressedFrames.shrink_to_fit();
+	_compressBuffer = QByteArray();
+}
+
+int CacheState::headerSize() const {
+	return 8 * sizeof(qint32);
 }
 
 void CacheState::writeHeader() {
-	Expects(_framesReady == 0);
 	Expects(_data.isEmpty());
 
 	QDataStream stream(&_data, QIODevice::WriteOnly);
 
 	stream
-		<< static_cast<quint8>(Encoder::YUV420A4_LZ4)
+		<< static_cast<qint32>(Encoder::YUV420A4_LZ4)
 		<< _size
 		<< _original
 		<< qint32(_frameRate)
 		<< qint32(_framesCount)
-		<< qint32(++_framesReady);
+		<< qint32(_framesReady);
-	_headerSize = stream.device()->pos();
-}
-
-void CacheState::incrementFramesReady() {
-	Expects(_headerSize > sizeof(qint32) && _data.size() > _headerSize);
-
-	const auto framesReady = qint32(++_framesReady);
-	bytes::copy(
-		bytes::make_detached_span(_data).subspan(
-			_headerSize - sizeof(qint32)),
-		bytes::object_as_span(&framesReady));
 }
 
 void CacheState::writeCompressedDelta() {
-	auto length = qint32(0);
+	CompressFromRaw(_compressBuffer, _uncompressed);
-	const auto size = _data.size();
+	_compressedFrames.push_back(_compressBuffer);
-	_data.resize(size + sizeof(length));
+	_compressedFrames.back().detach();
-	CompressFromRaw(_data, _uncompressed);
-	length = _data.size() - size - sizeof(length);
-	bytes::copy(
-		bytes::make_detached_span(_data).subspan(size),
-		bytes::object_as_span(&length));
 }
 
 void CacheState::prepareBuffers() {

Telegram/SourceFiles/lottie/lottie_cache.h

@@ -48,7 +48,7 @@ private:
 
 class CacheState {
 public:
-	enum class Encoder : quint8 {
+	enum class Encoder : qint8 {
 		YUV420A4_LZ4,
 	};
 
@@ -75,15 +75,18 @@ public:
 		int index);
 
 private:
+	int headerSize() const;
 	void prepareBuffers();
+	void finalizeEncoding();
 
 	void writeHeader();
-	void incrementFramesReady();
 	[[nodiscard]] bool readHeader(const FrameRequest &request);
 	void writeCompressedDelta();
 	[[nodiscard]] bool readCompressedDelta();
 
 	QByteArray _data;
+	std::vector<QByteArray> _compressedFrames;
+	QByteArray _compressBuffer;
 	QSize _size;
 	QSize _original;
 	AlignedStorage _uncompressed;
@@ -92,7 +95,6 @@ private:
 	int _frameRate = 0;
 	int _framesCount = 0;
 	int _framesReady = 0;
-	int _headerSize = 0;
 	int _offset = 0;
 	int _offsetFrameIndex = 0;
 	Encoder _encoder = Encoder::YUV420A4_LZ4;

Telegram/gyp/lib_lottie.gyp

@@ -24,6 +24,7 @@
       'official_build_target%': '',
       'submodules_loc': '../ThirdParty',
       'rlottie_loc': '<(submodules_loc)/rlottie/inc',
+      'lz4_loc': '<(submodules_loc)/lz4/lib',
     },
     'dependencies': [
       'crl.gyp:crl',
@@ -31,6 +32,7 @@
       'lib_rlottie.gyp:lib_rlottie',
       'lib_storage.gyp:lib_storage',
       'lib_ffmpeg.gyp:lib_ffmpeg',
+      'lib_lz4.gyp:lib_lz4',
     ],
     'export_dependent_settings': [
       'crl.gyp:crl',
@@ -38,6 +40,7 @@
       'lib_rlottie.gyp:lib_rlottie',
       'lib_storage.gyp:lib_storage',
       'lib_ffmpeg.gyp:lib_ffmpeg',
+      'lib_lz4.gyp:lib_lz4',
     ],
     'defines': [
       'LOT_BUILD',
@@ -49,6 +52,7 @@
       '<(libs_loc)/zlib',
       '<(libs_loc)/ffmpeg',
       '<(rlottie_loc)',
+      '<(lz4_loc)',
       '<(submodules_loc)/GSL/include',
       '<(submodules_loc)/variant/include',
       '<(submodules_loc)/crl/src',

Telegram/gyp/lib_lz4.gyp

@@ -0,0 +1,48 @@
+# This file is part of Telegram Desktop,
+# the official desktop application for the Telegram messaging service.
+#
+# For license and copyright information please follow this link:
+# https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+
+{
+  'includes': [
+    'common.gypi',
+  ],
+  'targets': [{
+    'target_name': 'lib_lz4',
+    'type': 'static_library',
+    'includes': [
+      'common.gypi',
+      'telegram_linux.gypi',
+    ],
+    'variables': {
+      'official_build_target%': '',
+      'submodules_loc': '../ThirdParty',
+      'lz4_loc': '<(submodules_loc)/lz4/lib',
+    },
+    'defines': [
+    ],
+    'include_dirs': [
+      '<(lz4_loc)',
+    ],
+    'sources': [
+      '<(lz4_loc)/lz4.c',
+      '<(lz4_loc)/lz4.h',
+      '<(lz4_loc)/lz4frame.c',
+      '<(lz4_loc)/lz4frame.h',
+      '<(lz4_loc)/lz4frame_static.h',
+      '<(lz4_loc)/lz4hc.c',
+      '<(lz4_loc)/lz4hc.h',
+      '<(lz4_loc)/xxhash.c',
+      '<(lz4_loc)/xxhash.h',
+    ],
+    'conditions': [[ 'build_macold', {
+      'xcode_settings': {
+        'OTHER_CPLUSPLUSFLAGS': [ '-nostdinc++' ],
+      },
+      'include_dirs': [
+        '/usr/local/macold/include/c++/v1',
+      ],
+    }]],
+  }],
+}