From e2eb9cea00e7984d85f589dfc1283eb08cea9c79 Mon Sep 17 00:00:00 2001
From: John Preston <johnprestonmail@gmail.com>
Date: Tue, 5 Mar 2019 13:00:49 +0400
Subject: [PATCH] Apply sample_aspect_ratio in streaming.
---
.../media/streaming/media_streaming_file.cpp | 1 +
.../media/streaming/media_streaming_player.h | 1 +
.../streaming/media_streaming_utility.cpp | 20 +++++++-
.../media/streaming/media_streaming_utility.h | 4 ++
.../streaming/media_streaming_video_track.cpp | 3 +-
.../streaming/media_streaming_video_track.h | 1 +
.../media/view/media_view_overlay_widget.cpp | 48 ++++++++++++-------
7 files changed, 58 insertions(+), 20 deletions(-)
diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_file.cpp b/Telegram/SourceFiles/media/streaming/media_streaming_file.cpp
index 8cc0fe7e1..6048ba3cc 100644
--- a/Telegram/SourceFiles/media/streaming/media_streaming_file.cpp
+++ b/Telegram/SourceFiles/media/streaming/media_streaming_file.cpp
@@ -111,6 +111,7 @@ Stream File::Context::initStream(AVMediaType type) {
const auto info = _format->streams[index];
if (type == AVMEDIA_TYPE_VIDEO) {
result.rotation = ReadRotationFromMetadata(info);
+ result.aspect = ValidateAspectRatio(info->sample_aspect_ratio);
} else if (type == AVMEDIA_TYPE_AUDIO) {
result.frequency = info->codecpar->sample_rate;
if (!result.frequency) {
diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_player.h b/Telegram/SourceFiles/media/streaming/media_streaming_player.h
index 2471377f6..3794a70bc 100644
--- a/Telegram/SourceFiles/media/streaming/media_streaming_player.h
+++ b/Telegram/SourceFiles/media/streaming/media_streaming_player.h
@@ -59,6 +59,7 @@ public:
[[nodiscard]] rpl::producer<Update, Error> updates() const;
[[nodiscard]] QImage frame(const FrameRequest &request) const;
+ //[[nodiscard]] int videoRotation() const;
[[nodiscard]] Media::Player::TrackState prepareLegacyState() const;
diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_utility.cpp b/Telegram/SourceFiles/media/streaming/media_streaming_utility.cpp
index 3d046f2f7..cfd07fd44 100644
--- a/Telegram/SourceFiles/media/streaming/media_streaming_utility.cpp
+++ b/Telegram/SourceFiles/media/streaming/media_streaming_utility.cpp
@@ -23,17 +23,25 @@ constexpr auto kAlignImageBy = 16;
constexpr auto kPixelBytesSize = 4;
constexpr auto kImageFormat = QImage::Format_ARGB32_Premultiplied;
constexpr auto kAvioBlockSize = 4096;
+constexpr auto kMaxScaleByAspectRatio = 16;
void AlignedImageBufferCleanupHandler(void* data) {
const auto buffer = static_cast<uchar*>(data);
delete[] buffer;
}
-bool IsAlignedImage(const QImage &image) {
+[[nodiscard]] bool IsAlignedImage(const QImage &image) {
return !(reinterpret_cast<uintptr_t>(image.bits()) % kAlignImageBy)
&& !(image.bytesPerLine() % kAlignImageBy);
}
+[[nodiscard]] bool IsValidAspectRatio(AVRational aspect) {
+ return (aspect.num > 0)
+ && (aspect.den > 0)
+ && (aspect.num <= aspect.den * kMaxScaleByAspectRatio)
+ && (aspect.den <= aspect.num * kMaxScaleByAspectRatio);
+}
+
} // namespace
bool GoodStorageForFrame(const QImage &storage, QSize size) {
@@ -303,6 +311,16 @@ int ReadRotationFromMetadata(not_null<AVStream*> stream) {
return 0;
}
+AVRational ValidateAspectRatio(AVRational aspect) {
+ return IsValidAspectRatio(aspect) ? aspect : kNormalAspect;
+}
+
+QSize CorrectByAspect(QSize size, AVRational aspect) {
+ Expects(IsValidAspectRatio(aspect));
+
+ return QSize(size.width() * aspect.num / aspect.den, size.height());
+}
+
bool RotationSwapWidthHeight(int rotation) {
return (rotation == 90 || rotation == 270);
}
diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_utility.h b/Telegram/SourceFiles/media/streaming/media_streaming_utility.h
index 6b0707171..8f16d1656 100644
--- a/Telegram/SourceFiles/media/streaming/media_streaming_utility.h
+++ b/Telegram/SourceFiles/media/streaming/media_streaming_utility.h
@@ -19,6 +19,7 @@ namespace Media {
namespace Streaming {
constexpr auto kUniversalTimeBase = AVRational{ 1, AV_TIME_BASE };
+constexpr auto kNormalAspect = AVRational{ 1, 1 };
struct TimePoint {
crl::time trackTime = kTimeUnknown;
@@ -176,6 +177,7 @@ struct Stream {
// Video only.
int rotation = 0;
+ AVRational aspect = kNormalAspect;
SwscalePointer swscale;
};
@@ -191,7 +193,9 @@ void LogError(QLatin1String method, AvErrorWrap error);
AVRational timeBase);
[[nodiscard]] crl::time FramePosition(const Stream &stream);
[[nodiscard]] int ReadRotationFromMetadata(not_null<AVStream*> stream);
+[[nodiscard]] AVRational ValidateAspectRatio(AVRational aspect);
[[nodiscard]] bool RotationSwapWidthHeight(int rotation);
+[[nodiscard]] QSize CorrectByAspect(QSize size, AVRational aspect);
[[nodiscard]] AvErrorWrap ProcessPacket(Stream &stream, Packet &&packet);
[[nodiscard]] AvErrorWrap ReadNextFrame(Stream &stream);
diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_video_track.cpp b/Telegram/SourceFiles/media/streaming/media_streaming_video_track.cpp
index c8a0ddc01..fdfa68247 100644
--- a/Telegram/SourceFiles/media/streaming/media_streaming_video_track.cpp
+++ b/Telegram/SourceFiles/media/streaming/media_streaming_video_track.cpp
@@ -394,7 +394,7 @@ void VideoTrackObject::callReady() {
Assert(frame != nullptr);
auto data = VideoInformation();
- data.size = frame->original.size();
+ data.size = CorrectByAspect(frame->original.size(), _stream.aspect);
if (RotationSwapWidthHeight(_stream.rotation)) {
data.size.transpose();
}
@@ -586,6 +586,7 @@ VideoTrack::VideoTrack(
, _streamTimeBase(stream.timeBase)
, _streamDuration(stream.duration)
//, _streamRotation(stream.rotation)
+//, _streamAspect(stream.aspect)
, _shared(std::make_unique<Shared>())
, _wrapped(
options,
diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_video_track.h b/Telegram/SourceFiles/media/streaming/media_streaming_video_track.h
index 5dc415ed1..f5215219e 100644
--- a/Telegram/SourceFiles/media/streaming/media_streaming_video_track.h
+++ b/Telegram/SourceFiles/media/streaming/media_streaming_video_track.h
@@ -123,6 +123,7 @@ private:
const AVRational _streamTimeBase;
const crl::time _streamDuration = 0;
//const int _streamRotation = 0;
+ //AVRational _streamAspect = kNormalAspect;
std::unique_ptr<Shared> _shared;
using Implementation = VideoTrackObject;
diff --git a/Telegram/SourceFiles/media/view/media_view_overlay_widget.cpp b/Telegram/SourceFiles/media/view/media_view_overlay_widget.cpp
index 36ac74b76..528f384ef 100644
--- a/Telegram/SourceFiles/media/view/media_view_overlay_widget.cpp
+++ b/Telegram/SourceFiles/media/view/media_view_overlay_widget.cpp
@@ -1199,7 +1199,15 @@ void OverlayWidget::onCopy() {
if (!_current.isNull()) {
QApplication::clipboard()->setPixmap(_current);
} else if (videoShown()) {
- QApplication::clipboard()->setImage(videoFrame());
+ // #TODO streaming later apply rotation
+ auto image = videoFrame();
+ if (image.size() != _streamed->info.video.size) {
+ image = image.scaled(
+ _streamed->info.video.size,
+ Qt::IgnoreAspectRatio,
+ Qt::SmoothTransformation);
+ }
+ QApplication::clipboard()->setImage(std::move(image));
}
} else {
if (!_photo || !_photo->loaded()) return;
@@ -1900,13 +1908,14 @@ void OverlayWidget::initStreamingThumbnail() {
} else if (thumb && !useThumb) {
thumb->load(fileOrigin());
}
+ const auto size = useGood ? good->size() : _doc->dimensions;
if (!useGood && !thumb && !blurred) {
return;
- } else if (_doc->dimensions.isEmpty()) {
+ } else if (size.isEmpty()) {
return;
}
- const auto w = _doc->dimensions.width();
- const auto h = _doc->dimensions.height();
+ const auto w = size.width();
+ const auto h = size.height();
const auto options = VideoThumbOptions(_doc);
const auto goodOptions = (options & ~Images::Option::Blurred);
_current = (useGood
@@ -1962,10 +1971,17 @@ void OverlayWidget::validateStreamedGoodThumbnail() {
Expects(_doc != nullptr);
const auto good = _doc->goodThumbnail();
- const auto &image = _streamed->info.video.cover;
+ auto image = _streamed->info.video.cover;
if (image.isNull() || (good && good->loaded()) || _doc->uploading()) {
return;
}
+ // #TODO streaming later apply rotation
+ if (image.size() != _streamed->info.video.size) {
+ image = image.scaled(
+ _streamed->info.video.size,
+ Qt::IgnoreAspectRatio,
+ Qt::SmoothTransformation);
+ }
auto bytes = QByteArray();
{
auto buffer = QBuffer(&bytes);
@@ -1976,7 +1992,7 @@ void OverlayWidget::validateStreamedGoodThumbnail() {
LOG(("App Error: Bad thumbnail data for saving to cache."));
} else if (_doc->uploading()) {
_doc->setGoodThumbnailOnUpload(
- base::duplicate(image),
+ std::move(image),
std::move(bytes));
} else {
_doc->owner().cache().putIfEmpty(
@@ -2187,7 +2203,7 @@ void OverlayWidget::restartAtSeekPosition(crl::time position) {
}
auto options = Streaming::PlaybackOptions();
options.position = position;
- if (_doc->isAnimation() || true) {
+ if (_doc->isAnimation()) {
options.mode = Streaming::Mode::Video;
options.loop = true;
}
@@ -2348,17 +2364,13 @@ void OverlayWidget::paintEvent(QPaintEvent *e) {
if (rect.intersects(r)) {
if (videoShown()) {
const auto image = videoFrame();
- if (image.width() != _w) {
- //if (_fullScreenVideo) {
- // const auto fill = rect.intersected(this->rect());
- // PaintImageProfile(p, image, rect, fill);
- //} else {
- PainterHighQualityEnabler hq(p);
- p.drawImage(rect, image);
- //}
- } else {
- p.drawImage(rect.topLeft(), image);
- }
+ //if (_fullScreenVideo) {
+ // const auto fill = rect.intersected(this->rect());
+ // PaintImageProfile(p, image, rect, fill);
+ //} else {
+ PainterHighQualityEnabler hq(p);
+ p.drawImage(rect, image);
+ //}
} else if (!_current.isNull()) {
if ((!_doc || !_doc->getStickerLarge()) && _current.hasAlpha()) {
p.fillRect(rect, _transparentBrush);