From 3e9b811875d1b1fdff686c195f6c3a47de209d41 Mon Sep 17 00:00:00 2001
From: John Preston <johnprestonmail@gmail.com>
Date: Fri, 22 Feb 2019 16:39:32 +0400
Subject: [PATCH] Implement precise seek in streaming.
---
Telegram/SourceFiles/data/data_document.cpp | 11 +++-
.../streaming/media_streaming_audio_track.cpp | 34 +++++++++---
.../streaming/media_streaming_audio_track.h | 4 ++
.../media/streaming/media_streaming_file.cpp | 12 ++---
.../media/streaming/media_streaming_utility.h | 1 -
.../streaming/media_streaming_video_track.cpp | 54 ++++++++-----------
6 files changed, 70 insertions(+), 46 deletions(-)
diff --git a/Telegram/SourceFiles/data/data_document.cpp b/Telegram/SourceFiles/data/data_document.cpp
index fd8c97f4b..8886719cb 100644
--- a/Telegram/SourceFiles/data/data_document.cpp
+++ b/Telegram/SourceFiles/data/data_document.cpp
@@ -302,6 +302,7 @@ void StartStreaming(
static auto options = Media::Streaming::PlaybackOptions();
static auto speed = 1.;
static auto step = pow(2., 1. / 12);
+ static auto frame = QImage();
class Panel
#if defined Q_OS_MAC && !defined OS_MAC_OLD
@@ -339,6 +340,9 @@ void StartStreaming(
player->pause();
}
void mouseReleaseEvent(QMouseEvent *e) override {
+ if (player->ready()) {
+ frame = player->frame({});
+ }
preloaded = position = options.position = std::clamp(
(duration * e->pos().x()) / width(),
crl::time(0),
@@ -364,6 +368,7 @@ void StartStreaming(
options.speed = speed;
//options.syncVideoByAudio = false;
preloaded = position = options.position = 0;
+ frame = QImage();
player->play(options);
player->updates(
) | rpl::start_with_next_error_done([=](Update &&update) {
@@ -391,7 +396,11 @@ void StartStreaming(
if (player->ready()) {
Painter(video.get()).drawImage(
video->rect(),
- player->frame(FrameRequest()));
+ player->frame({}));
+ } else if (!frame.isNull()) {
+ Painter(video.get()).drawImage(
+ video->rect(),
+ frame);
} else {
Painter(video.get()).fillRect(
rect,
diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_audio_track.cpp b/Telegram/SourceFiles/media/streaming/media_streaming_audio_track.cpp
index 44be38523..8720707ab 100644
--- a/Telegram/SourceFiles/media/streaming/media_streaming_audio_track.cpp
+++ b/Telegram/SourceFiles/media/streaming/media_streaming_audio_track.cpp
@@ -61,29 +61,51 @@ bool AudioTrack::initialized() const {
}
bool AudioTrack::tryReadFirstFrame(Packet &&packet) {
- // #TODO streaming fix seek to the end.
if (ProcessPacket(_stream, std::move(packet)).failed()) {
return false;
}
if (const auto error = ReadNextFrame(_stream)) {
if (error.code() == AVERROR_EOF) {
- // #TODO streaming fix seek to the end.
- return false;
+ if (!_initialSkippingFrame) {
+ return false;
+ }
+ // Return the last valid frame if we seek too far.
+ _stream.frame = std::move(_initialSkippingFrame);
+ return processFirstFrame();
} else if (error.code() != AVERROR(EAGAIN) || _noMoreData) {
return false;
+ } else {
+ // Waiting for more packets.
+ return true;
}
- return true;
} else if (!fillStateFromFrame()) {
return false;
+ } else if (_startedPosition < _options.position) {
+ // Seek was with AVSEEK_FLAG_BACKWARD so first we get old frames.
+ // Try skipping frames until one is after the requested position.
+ std::swap(_initialSkippingFrame, _stream.frame);
+ if (!_stream.frame) {
+ _stream.frame = MakeFramePointer();
+ }
+ return true;
+ } else {
+ return processFirstFrame();
}
+}
+
+bool AudioTrack::processFirstFrame() {
mixerInit();
callReady();
return true;
}
bool AudioTrack::fillStateFromFrame() {
- _startedPosition = FramePosition(_stream);
- return (_startedPosition != kTimeUnknown);
+ const auto position = FramePosition(_stream);
+ if (position == kTimeUnknown) {
+ return false;
+ }
+ _startedPosition = position;
+ return true;
}
void AudioTrack::mixerInit() {
diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_audio_track.h b/Telegram/SourceFiles/media/streaming/media_streaming_audio_track.h
index c6e98936e..8f49025b2 100644
--- a/Telegram/SourceFiles/media/streaming/media_streaming_audio_track.h
+++ b/Telegram/SourceFiles/media/streaming/media_streaming_audio_track.h
@@ -52,6 +52,7 @@ private:
[[nodiscard]] bool initialized() const;
[[nodiscard]] bool tryReadFirstFrame(Packet &&packet);
[[nodiscard]] bool fillStateFromFrame();
+ [[nodiscard]] bool processFirstFrame();
void mixerInit();
void mixerEnqueue(Packet &&packet);
void mixerForceToBuffer();
@@ -78,6 +79,9 @@ private:
// After that accessed from the main thread.
rpl::variable<crl::time> _playPosition;
+ // For initial frame skipping for an exact seek.
+ FramePointer _initialSkippingFrame;
+
};
} // namespace Streaming
diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_file.cpp b/Telegram/SourceFiles/media/streaming/media_streaming_file.cpp
index 53d662a9a..ca706b655 100644
--- a/Telegram/SourceFiles/media/streaming/media_streaming_file.cpp
+++ b/Telegram/SourceFiles/media/streaming/media_streaming_file.cpp
@@ -110,15 +110,8 @@ Stream File::Context::initStream(AVMediaType type) {
}
const auto info = _formatContext->streams[index];
- result.codec = MakeCodecPointer(info);
- if (!result.codec) {
- return {};
- }
-
if (type == AVMEDIA_TYPE_VIDEO) {
- const auto codec = result.codec.get();
result.rotation = ReadRotationFromMetadata(info);
- result.dimensions = QSize(codec->width, codec->height);
} else if (type == AVMEDIA_TYPE_AUDIO) {
result.frequency = info->codecpar->sample_rate;
if (!result.frequency) {
@@ -126,6 +119,11 @@ Stream File::Context::initStream(AVMediaType type) {
}
}
+ result.codec = MakeCodecPointer(info);
+ if (!result.codec) {
+ return {};
+ }
+
result.frame = MakeFramePointer();
if (!result.frame) {
return {};
diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_utility.h b/Telegram/SourceFiles/media/streaming/media_streaming_utility.h
index 72b083230..d769f5d0c 100644
--- a/Telegram/SourceFiles/media/streaming/media_streaming_utility.h
+++ b/Telegram/SourceFiles/media/streaming/media_streaming_utility.h
@@ -154,7 +154,6 @@ struct Stream {
// Video only.
int rotation = 0;
- QSize dimensions;
SwsContextPointer swsContext;
};
diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_video_track.cpp b/Telegram/SourceFiles/media/streaming/media_streaming_video_track.cpp
index 8e503b0cf..04863a54a 100644
--- a/Telegram/SourceFiles/media/streaming/media_streaming_video_track.cpp
+++ b/Telegram/SourceFiles/media/streaming/media_streaming_video_track.cpp
@@ -47,10 +47,7 @@ private:
[[nodiscard]] bool interrupted() const;
[[nodiscard]] bool tryReadFirstFrame(Packet &&packet);
[[nodiscard]] bool fillStateFromFrame();
- [[nodiscard]] bool fillStateFromFakeLastFrame();
- [[nodiscard]] bool fillStateFromFrameTime(crl::time frameTime);
- [[nodiscard]] QImage createFakeLastFrame() const;
- [[nodiscard]] bool processFirstFrame(QImage frame);
+ [[nodiscard]] bool processFirstFrame();
void queueReadFrames(crl::time delay = 0);
void readFrames();
[[nodiscard]] bool readFrame(not_null<Frame*> frame);
@@ -83,6 +80,9 @@ private:
bool _queued = false;
base::ConcurrentTimer _readFramesTimer;
+ // For initial frame skipping for an exact seek.
+ FramePointer _initialSkippingFrame;
+
};
VideoTrackObject::VideoTrackObject(
@@ -259,10 +259,12 @@ bool VideoTrackObject::tryReadFirstFrame(Packet &&packet) {
auto frame = QImage();
if (const auto error = ReadNextFrame(_stream)) {
if (error.code() == AVERROR_EOF) {
- if (!fillStateFromFakeLastFrame()) {
+ if (!_initialSkippingFrame) {
return false;
}
- return processFirstFrame(createFakeLastFrame());
+ // Return the last valid frame if we seek too far.
+ _stream.frame = std::move(_initialSkippingFrame);
+ return processFirstFrame();
} else if (error.code() != AVERROR(EAGAIN) || _noMoreData) {
return false;
} else {
@@ -271,22 +273,21 @@ bool VideoTrackObject::tryReadFirstFrame(Packet &&packet) {
}
} else if (!fillStateFromFrame()) {
return false;
+ } else if (_syncTimePoint.trackTime < _options.position) {
+ // Seek was with AVSEEK_FLAG_BACKWARD so first we get old frames.
+ // Try skipping frames until one is after the requested position.
+ std::swap(_initialSkippingFrame, _stream.frame);
+ if (!_stream.frame) {
+ _stream.frame = MakeFramePointer();
+ }
+ return true;
+ } else {
+ return processFirstFrame();
}
- return processFirstFrame(ConvertFrame(_stream, QSize(), QImage()));
}
-QImage VideoTrackObject::createFakeLastFrame() const {
- if (_stream.dimensions.isEmpty()) {
- LOG(("Streaming Error: Can't seek to the end of the video "
- "in case the codec doesn't provide valid dimensions."));
- return QImage();
- }
- auto result = CreateImageForOriginalFrame(_stream.dimensions);
- result.fill(Qt::black);
- return result;
-}
-
-bool VideoTrackObject::processFirstFrame(QImage frame) {
+bool VideoTrackObject::processFirstFrame() {
+ auto frame = ConvertFrame(_stream, QSize(), QImage());
if (frame.isNull()) {
return false;
}
@@ -312,20 +313,11 @@ crl::time VideoTrackObject::currentFramePosition() const {
}
bool VideoTrackObject::fillStateFromFrame() {
- return fillStateFromFrameTime(currentFramePosition());
-}
-
-bool VideoTrackObject::fillStateFromFakeLastFrame() {
- return fillStateFromFrameTime(_stream.duration);
-}
-
-bool VideoTrackObject::fillStateFromFrameTime(crl::time frameTime) {
- Expects(_syncTimePoint.trackTime == kTimeUnknown);
-
- if (frameTime == kTimeUnknown) {
+ const auto position = currentFramePosition();
+ if (position == kTimeUnknown) {
return false;
}
- _syncTimePoint.trackTime = frameTime;
+ _syncTimePoint.trackTime = position;
return true;
}