From 10772f4ac5080c94a89e65bb4ee2fa6f76ead57e Mon Sep 17 00:00:00 2001 From: John Preston Date: Wed, 26 Jun 2019 16:18:00 +0200 Subject: [PATCH] Started Lottie::CacheState class. --- .../SourceFiles/chat_helpers/stickers.cpp | 5 +- .../chat_helpers/stickers_list_widget.cpp | 6 +- .../history/media/history_media_sticker.cpp | 4 +- .../SourceFiles/lottie/lottie_animation.cpp | 162 +++++++++---- .../SourceFiles/lottie/lottie_animation.h | 25 +- Telegram/SourceFiles/lottie/lottie_cache.cpp | 225 ++++++++++++++++++ Telegram/SourceFiles/lottie/lottie_cache.h | 80 +++++++ .../lottie/lottie_frame_renderer.cpp | 4 +- .../lottie/lottie_frame_renderer.h | 4 + Telegram/SourceFiles/window/layer_widget.cpp | 4 +- Telegram/gyp/lib_lottie.gyp | 2 + 11 files changed, 449 insertions(+), 72 deletions(-) create mode 100644 Telegram/SourceFiles/lottie/lottie_cache.cpp create mode 100644 Telegram/SourceFiles/lottie/lottie_cache.h diff --git a/Telegram/SourceFiles/chat_helpers/stickers.cpp b/Telegram/SourceFiles/chat_helpers/stickers.cpp index 694a70905..83a41c908 100644 --- a/Telegram/SourceFiles/chat_helpers/stickers.cpp +++ b/Telegram/SourceFiles/chat_helpers/stickers.cpp @@ -1100,11 +1100,8 @@ std::unique_ptr LottieFromDocument( data, filepath, box); - } else if (!data.isEmpty()) { - return Lottie::FromData(data); - } else { - return Lottie::FromFile(filepath); } + return Lottie::FromContent(data, filepath); } } // namespace Stickers diff --git a/Telegram/SourceFiles/chat_helpers/stickers_list_widget.cpp b/Telegram/SourceFiles/chat_helpers/stickers_list_widget.cpp index 9c69ab10b..f885faad5 100644 --- a/Telegram/SourceFiles/chat_helpers/stickers_list_widget.cpp +++ b/Telegram/SourceFiles/chat_helpers/stickers_list_widget.cpp @@ -1368,9 +1368,9 @@ void StickersListWidget::setupLottie(Set &set, int section, int index) { auto &sticker = set.stickers[index]; const auto document = sticker.document; - sticker.animated = document->data().isEmpty() - ? Lottie::FromFile(document->filepath()) - : Lottie::FromData(document->data()); + sticker.animated = Lottie::FromContent( + document->data(), + document->filepath()); const auto animation = sticker.animated.get(); animation->updates( diff --git a/Telegram/SourceFiles/history/media/history_media_sticker.cpp b/Telegram/SourceFiles/history/media/history_media_sticker.cpp index 7dfa891cb..66c04ee02 100644 --- a/Telegram/SourceFiles/history/media/history_media_sticker.cpp +++ b/Telegram/SourceFiles/history/media/history_media_sticker.cpp @@ -97,9 +97,7 @@ QSize HistorySticker::countCurrentSize(int newWidth) { } void HistorySticker::setupLottie() { - _lottie = _data->data().isEmpty() - ? Lottie::FromFile(_data->filepath()) - : Lottie::FromData(_data->data()); + _lottie = Lottie::FromContent(_data->data(), _data->filepath()); _parent->data()->history()->owner().registerHeavyViewPart(_parent); _lottie->updates( diff --git a/Telegram/SourceFiles/lottie/lottie_animation.cpp b/Telegram/SourceFiles/lottie/lottie_animation.cpp index 12e79e664..d4b708432 100644 --- a/Telegram/SourceFiles/lottie/lottie_animation.cpp +++ b/Telegram/SourceFiles/lottie/lottie_animation.cpp @@ -8,6 +8,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include "lottie/lottie_animation.h" #include "lottie/lottie_frame_renderer.h" +#include "lottie/lottie_cache.h" #include "storage/cache/storage_cache_database.h" #include "base/algorithm.h" #include "zlib.h" @@ -56,19 +57,85 @@ std::string UnpackGzip(const QByteArray &bytes) { return result; } -} // namespace - -std::unique_ptr FromFile(const QString &path) { - return FromData([&] { - auto f = QFile(path); - return (f.size() <= kMaxFileSize && f.open(QIODevice::ReadOnly)) - ? f.readAll() - : QByteArray(); - }()); +QByteArray ReadFile(const QString &filepath) { + auto f = QFile(filepath); + return (f.size() <= kMaxFileSize && f.open(QIODevice::ReadOnly)) + ? f.readAll() + : QByteArray(); } -std::unique_ptr FromData(const QByteArray &data) { - return std::make_unique(base::duplicate(data)); +QByteArray ReadContent(const QByteArray &data, const QString &filepath) { + return data.isEmpty() ? ReadFile(filepath) : base::duplicate(data); +} + +std::optional ContentError(const QByteArray &content) { + if (content.size() > kMaxFileSize) { + qWarning() << "Lottie Error: Too large file: " << content.size(); + return Error::ParseFailed; + } + return std::nullopt; +} + +std::unique_ptr CreateImplementation( + const QByteArray &content) { + const auto string = UnpackGzip(content); + Assert(string.size() <= kMaxFileSize); + + auto result = rlottie::Animation::loadFromData(string, std::string()); + if (!result) { + qWarning() << "Lottie Error: Parse failed."; + } + return result; +} + +details::InitData CheckSharedState(std::unique_ptr state) { + Expects(state != nullptr); + + auto information = state->information(); + if (!information.frameRate + || information.framesCount <= 0 + || information.size.isEmpty()) { + return Error::NotSupported; + } + return state; +} + +details::InitData Init(const QByteArray &content) { + if (const auto error = ContentError(content)) { + return *error; + } + auto animation = CreateImplementation(content); + return animation + ? CheckSharedState(std::make_unique( + std::move(animation))) + : Error::ParseFailed; +} + +details::InitData Init( + const QByteArray &content, + not_null cache, + Storage::Cache::Key key, + const QByteArray &cached, + QSize box) { + if (const auto error = ContentError(content)) { + return *error; + } + auto state = CacheState(cached, box); + const auto prepare = !state.framesCount() + || (state.framesReady() < state.framesCount()); + auto animation = prepare ? CreateImplementation(content) : nullptr; + return (!prepare || animation) + ? CheckSharedState(std::make_unique( + std::move(animation))) + : Error::ParseFailed; +} + +} // namespace + +std::unique_ptr FromContent( + const QByteArray &data, + const QString &filepath) { + return std::make_unique(ReadContent(data, filepath)); } std::unique_ptr FromCached( @@ -77,40 +144,14 @@ std::unique_ptr FromCached( const QByteArray &data, const QString &filepath, QSize box) { - return data.isEmpty() - ? Lottie::FromFile(filepath) - : Lottie::FromData(data); + return std::make_unique( + cache, + key, + ReadContent(data, filepath), + box); } -auto Init(QByteArray &&content) --> base::variant, Error> { - if (content.size() > kMaxFileSize) { - qWarning() - << "Lottie Error: Too large file: " - << content.size(); - return Error::ParseFailed; - } - const auto string = UnpackGzip(content); - Assert(string.size() <= kMaxFileSize); - - auto animation = rlottie::Animation::loadFromData(string, std::string()); - if (!animation) { - qWarning() - << "Lottie Error: Parse failed."; - return Error::ParseFailed; - } - - auto result = std::make_unique(std::move(animation)); - auto information = result->information(); - if (!information.frameRate - || information.framesCount <= 0 - || information.size.isEmpty()) { - return Error::NotSupported; - } - return std::move(result); -} - -QImage ReadThumbnail(QByteArray &&content) { +QImage ReadThumbnail(const QByteArray &content) { return Init(std::move(content)).match([]( const std::unique_ptr &state) { return state->frameForPaint()->original; @@ -119,15 +160,28 @@ QImage ReadThumbnail(QByteArray &&content) { }); } -Animation::Animation(QByteArray &&content) +Animation::Animation(const QByteArray &content) : _timer([=] { checkNextFrameRender(); }) { const auto weak = base::make_weak(this); - crl::async([=, content = base::take(content)]() mutable { - crl::on_main(weak, [this, result = Init(std::move(content))]() mutable { - result.match([&](std::unique_ptr &state) { - parseDone(std::move(state)); - }, [&](Error error) { - parseFailed(error); + crl::async([=] { + crl::on_main(weak, [=, data = Init(content)]() mutable { + initDone(std::move(data)); + }); + }); +} + +Animation::Animation( + not_null cache, + Storage::Cache::Key key, + const QByteArray &content, + QSize box) +: _timer([=] { checkNextFrameRender(); }) { + const auto weak = base::make_weak(this); + cache->get(key, [=](QByteArray &&cached) mutable { + crl::async([=] { + auto result = Init(content, cache, key, cached, box); + crl::on_main(weak, [=, data = std::move(result)]() mutable { + initDone(std::move(data)); }); }); }); @@ -140,6 +194,14 @@ Animation::~Animation() { } } +void Animation::initDone(details::InitData &&data) { + data.match([&](std::unique_ptr &state) { + parseDone(std::move(state)); + }, [&](Error error) { + parseFailed(error); + }); +} + void Animation::parseDone(std::unique_ptr state) { Expects(state != nullptr); diff --git a/Telegram/SourceFiles/lottie/lottie_animation.h b/Telegram/SourceFiles/lottie/lottie_animation.h index 68f32bca4..78595fa87 100644 --- a/Telegram/SourceFiles/lottie/lottie_animation.h +++ b/Telegram/SourceFiles/lottie/lottie_animation.h @@ -32,14 +32,15 @@ struct Key; namespace Lottie { -constexpr auto kMaxFileSize = 1024 * 1024; +inline constexpr auto kMaxFileSize = 1024 * 1024; -class Animation; class SharedState; +class Animation; class FrameRenderer; -std::unique_ptr FromFile(const QString &path); -std::unique_ptr FromData(const QByteArray &data); +std::unique_ptr FromContent( + const QByteArray &data, + const QString &filepath); std::unique_ptr FromCached( not_null cache, Storage::Cache::Key key, @@ -47,11 +48,22 @@ std::unique_ptr FromCached( const QString &filepath, QSize box); -QImage ReadThumbnail(QByteArray &&content); +QImage ReadThumbnail(const QByteArray &content); + +namespace details { + +using InitData = base::variant, Error>; + +} // namespace details class Animation final : public base::has_weak_ptr { public: - explicit Animation(QByteArray &&content); + explicit Animation(const QByteArray &content); + Animation( + not_null cache, + Storage::Cache::Key key, + const QByteArray &content, + QSize box); ~Animation(); //void play(const PlaybackOptions &options); @@ -69,6 +81,7 @@ public: void checkStep(); private: + void initDone(details::InitData &&data); void parseDone(std::unique_ptr state); void parseFailed(Error error); diff --git a/Telegram/SourceFiles/lottie/lottie_cache.cpp b/Telegram/SourceFiles/lottie/lottie_cache.cpp new file mode 100644 index 000000000..889c1f45a --- /dev/null +++ b/Telegram/SourceFiles/lottie/lottie_cache.cpp @@ -0,0 +1,225 @@ +/* +This file is part of Telegram Desktop, +the official desktop application for the Telegram messaging service. + +For license and copyright information please follow this link: +https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL +*/ +#include "lottie/lottie_cache.h" + +#include "lottie/lottie_frame_renderer.h" +#include "base/bytes.h" + +#include + +namespace Lottie { +namespace { + +constexpr auto kAlignStorage = 16; + +bool UncompressToRaw(AlignedStorage &to, bytes::const_span from) { + if (from.empty() || from.size() > to.rawSize()) { + return false; + } else if (from.size() == to.rawSize()) { + memcpy(to.raw(), from.data(), from.size()); + return true; + } else { + // #TODO stickers + return false; + } +} + +void Decode(QImage &to, const AlignedStorage &from, const QSize &fromSize) { + auto fromBytes = static_cast(from.aligned()); + auto toBytes = to.bits(); + const auto fromPerLine = from.bytesPerLine(); + const auto toPerLine = to.bytesPerLine(); + for (auto i = 0; i != to.height(); ++i) { + memcpy(toBytes, fromBytes, to.width() * 4); + fromBytes += fromPerLine; + toBytes += toPerLine; + } +} + +} // namespace + +void AlignedStorage::allocate(int packedBytesPerLine, int lines) { + Expects(packedBytesPerLine >= 0); + Expects(lines >= 0); + + _packedBytesPerLine = packedBytesPerLine; + _lines = lines; + reallocate(); +} + +void AlignedStorage::reallocate() { + const auto perLine = bytesPerLine(); + const auto total = perLine * _lines; + _buffer = QByteArray(total + kAlignStorage - 1, Qt::Uninitialized); + _raw = (perLine != _packedBytesPerLine) + ? QByteArray(_packedBytesPerLine * _lines, Qt::Uninitialized) + : QByteArray(); +} + +int AlignedStorage::lines() const { + return _lines; +} + +int AlignedStorage::rawSize() const { + return _lines * _packedBytesPerLine; +} + +void *AlignedStorage::raw() { + return (bytesPerLine() == _packedBytesPerLine) ? aligned() : _raw.data(); +} + +const void *AlignedStorage::raw() const { + return (bytesPerLine() == _packedBytesPerLine) ? aligned() : _raw.data(); +} + +int AlignedStorage::bytesPerLine() const { + return kAlignStorage + * ((_packedBytesPerLine + kAlignStorage - 1) / kAlignStorage); +} + +void *AlignedStorage::aligned() { + const auto result = reinterpret_cast(_buffer.data()); + return reinterpret_cast(kAlignStorage + * ((result + kAlignStorage - 1) / kAlignStorage)); +} + +const void *AlignedStorage::aligned() const { + const auto result = reinterpret_cast(_buffer.data()); + return reinterpret_cast(kAlignStorage + * ((result + kAlignStorage - 1) / kAlignStorage)); +} + +void AlignedStorage::copyRawToAligned() { + const auto fromPerLine = _packedBytesPerLine; + const auto toPerLine = bytesPerLine(); + if (fromPerLine == toPerLine) { + return; + } + auto from = static_cast(raw()); + auto to = static_cast(aligned()); + for (auto i = 0; i != _lines; ++i) { + memcpy(from, to, fromPerLine); + from += fromPerLine; + to += toPerLine; + } +} + +void AlignedStorage::copyAlignedToRaw() { + const auto fromPerLine = bytesPerLine(); + const auto toPerLine = _packedBytesPerLine; + if (fromPerLine == toPerLine) { + return; + } + auto from = static_cast(aligned()); + auto to = static_cast(raw()); + for (auto i = 0; i != _lines; ++i) { + memcpy(from, to, toPerLine); + from += fromPerLine; + to += toPerLine; + } +} + +CacheState::CacheState(const QByteArray &data, QSize box) +: _data(data) { + if (!readHeader(box)) { + _framesReady = 0; + } +} + +int CacheState::frameRate() const { + return _frameRate; +} + +int CacheState::framesReady() const { + return _framesReady; +} + +int CacheState::framesCount() const { + return _framesCount; +} + +bool CacheState::readHeader(QSize box) { + if (_data.isEmpty()) { + return false; + + } + QDataStream stream(&_data, QIODevice::ReadOnly); + + auto encoder = uchar(0); + stream >> encoder; + if (static_cast(encoder) != Encoder::YUV420A4_LZ4) { + return false; + } + auto size = QSize(); + auto original = QSize(); + auto frameRate = qint32(0); + auto framesCount = qint32(0); + auto framesReady = qint32(0); + stream + >> size + >> original + >> frameRate + >> framesCount + >> framesReady; + if (stream.status() != QDataStream::Ok + || original.isEmpty() + || (original.width() > kMaxSize) + || (original.height() > kMaxSize) + || (frameRate <= 0) + || (frameRate > kMaxFrameRate) + || (framesCount <= 0) + || (framesCount > kMaxFramesCount) + || (framesReady <= 0) + || (framesReady > framesCount) + || FrameRequest{ box }.size(original) != size) { + return false; + } + _size = size; + _original = original; + _frameRate = frameRate; + _framesCount = framesCount; + _framesReady = framesReady; + prepareBuffers(); + if (!readCompressedDelta(stream.device()->pos())) { + return false; + } + _uncompressed.copyRawToAligned(); + std::swap(_uncompressed, _previous); + Decode(_firstFrame, _previous, _size); + return true; +} + +QImage CacheState::takeFirstFrame() { + return std::move(_firstFrame); +} + +void CacheState::prepareBuffers() { + _uncompressed.allocate(_size.width() * 4, _size.height()); +} + +int CacheState::uncompressedDeltaSize() const { + return _size.width() * _size.height() * 4; // #TODO stickers +} + +bool CacheState::readCompressedDelta(int offset) { + auto length = qint32(0); + const auto part = bytes::make_span(_data).subspan(offset); + if (part.size() < sizeof(length)) { + return false; + } + bytes::copy(bytes::object_as_span(&length), part); + const auto bytes = part.subspan(sizeof(length)); + const auto uncompressedSize = uncompressedDeltaSize(); + + _offset = offset + length; + return (length <= bytes.size()) + ? UncompressToRaw(_uncompressed, bytes.subspan(0, length)) + : false; +} + +} // namespace Lottie diff --git a/Telegram/SourceFiles/lottie/lottie_cache.h b/Telegram/SourceFiles/lottie/lottie_cache.h new file mode 100644 index 000000000..db334c20d --- /dev/null +++ b/Telegram/SourceFiles/lottie/lottie_cache.h @@ -0,0 +1,80 @@ +/* +This file is part of Telegram Desktop, +the official desktop application for the Telegram messaging service. + +For license and copyright information please follow this link: +https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL +*/ +#pragma once + +#include +#include +#include + +namespace Lottie { + +class AlignedStorage { +public: + void allocate(int packedBytesPerLine, int lines); + + int lines() const; + int rawSize() const; + + // Gives a pointer to packedBytesPerLine * lines bytes of memory. + void *raw(); + const void *raw() const; + + // Gives a stride value in the aligned storage (% 16 == 0). + int bytesPerLine() const; + + // Gives a pointer to the aligned memory (% 16 == 0). + void *aligned(); + const void *aligned() const; + + void copyRawToAligned(); + void copyAlignedToRaw(); + +private: + void reallocate(); + + int _packedBytesPerLine = 0; + int _lines = 0; + QByteArray _raw; + QByteArray _buffer; + +}; + +class CacheState { +public: + enum class Encoder : uchar { + YUV420A4_LZ4, + }; + + CacheState(const QByteArray &data, QSize box); + + [[nodiscard]] int frameRate() const; + [[nodiscard]] int framesReady() const; + [[nodiscard]] int framesCount() const; + [[nodiscard]] QImage takeFirstFrame(); + +private: + [[nodiscard]] bool readHeader(QSize box); + void prepareBuffers(); + [[nodiscard]] bool readCompressedDelta(int offset); + [[nodiscard]] int uncompressedDeltaSize() const; + + QByteArray _data; + QSize _size; + QSize _original; + AlignedStorage _uncompressed; + AlignedStorage _previous; + QImage _firstFrame; + int _frameRate = 0; + int _framesCount = 0; + int _framesReady = 0; + int _offset = 0; + Encoder _encoder = Encoder::YUV420A4_LZ4; + +}; + +} // namespace Lottie diff --git a/Telegram/SourceFiles/lottie/lottie_frame_renderer.cpp b/Telegram/SourceFiles/lottie/lottie_frame_renderer.cpp index 1538a403e..8f34d87aa 100644 --- a/Telegram/SourceFiles/lottie/lottie_frame_renderer.cpp +++ b/Telegram/SourceFiles/lottie/lottie_frame_renderer.cpp @@ -23,8 +23,6 @@ namespace Lottie { namespace { constexpr auto kDisplaySkipped = crl::time(-1); -constexpr auto kMaxFrameRate = 120; -constexpr auto kMaxSize = 3096; std::weak_ptr GlobalInstance; @@ -199,7 +197,7 @@ void SharedState::calculateProperties() { (width > 0 && width < kMaxSize) ? int(width) : 0, (height > 0 && height < kMaxSize) ? int(height) : 0); _frameRate = (rate >= 1. && rate <= kMaxFrameRate) ? int(rate) : 0; - _framesCount = (count > 0) ? int(count) : 0; + _framesCount = (count > 0 && count <= kMaxFramesCount) ? int(count) : 0; } bool SharedState::isValid() const { diff --git a/Telegram/SourceFiles/lottie/lottie_frame_renderer.h b/Telegram/SourceFiles/lottie/lottie_frame_renderer.h index 883e6bdc7..df4212279 100644 --- a/Telegram/SourceFiles/lottie/lottie_frame_renderer.h +++ b/Telegram/SourceFiles/lottie/lottie_frame_renderer.h @@ -23,6 +23,10 @@ class Animation; namespace Lottie { +inline constexpr auto kMaxFrameRate = 120; +inline constexpr auto kMaxSize = 3096; +inline constexpr auto kMaxFramesCount = 600; + class Animation; class JsonObject; diff --git a/Telegram/SourceFiles/window/layer_widget.cpp b/Telegram/SourceFiles/window/layer_widget.cpp index a1accbc73..934e6bb9d 100644 --- a/Telegram/SourceFiles/window/layer_widget.cpp +++ b/Telegram/SourceFiles/window/layer_widget.cpp @@ -1049,9 +1049,7 @@ QSize MediaPreviewWidget::currentDimensions() const { void MediaPreviewWidget::setupLottie() { Expects(_document != nullptr); - _lottie = _document->data().isEmpty() - ? Lottie::FromFile(_document->filepath()) - : Lottie::FromData(_document->data()); + _lottie = Lottie::FromContent(_document->data(), _document->filepath()); _lottie->updates( ) | rpl::start_with_next_error([=](Lottie::Update update) { diff --git a/Telegram/gyp/lib_lottie.gyp b/Telegram/gyp/lib_lottie.gyp index a6ca9ede3..82fe6080d 100644 --- a/Telegram/gyp/lib_lottie.gyp +++ b/Telegram/gyp/lib_lottie.gyp @@ -53,6 +53,8 @@ 'sources': [ '<(src_loc)/lottie/lottie_animation.cpp', '<(src_loc)/lottie/lottie_animation.h', + '<(src_loc)/lottie/lottie_cache.cpp', + '<(src_loc)/lottie/lottie_cache.h', '<(src_loc)/lottie/lottie_common.h', '<(src_loc)/lottie/lottie_frame_renderer.cpp', '<(src_loc)/lottie/lottie_frame_renderer.h',