From 99e96a5b13d849e23fd47b0aaf310088891d1bcd Mon Sep 17 00:00:00 2001 From: John Preston Date: Tue, 5 Mar 2019 11:40:25 +0400 Subject: [PATCH] Allow looping video without audio in streaming. --- .../streaming/media_streaming_audio_track.cpp | 11 ++- .../streaming/media_streaming_audio_track.h | 1 + .../media/streaming/media_streaming_common.h | 1 + .../media/streaming/media_streaming_file.cpp | 29 ++++-- .../media/streaming/media_streaming_file.h | 1 + .../streaming/media_streaming_file_delegate.h | 4 +- .../streaming/media_streaming_player.cpp | 89 ++++++++++++------- .../media/streaming/media_streaming_player.h | 7 +- .../streaming/media_streaming_video_track.cpp | 69 +++++++++++--- .../streaming/media_streaming_video_track.h | 2 + .../media/view/media_view_overlay_widget.cpp | 5 +- 11 files changed, 162 insertions(+), 57 deletions(-) diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_audio_track.cpp b/Telegram/SourceFiles/media/streaming/media_streaming_audio_track.cpp index 0fa245508..acb4b05fc 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_audio_track.cpp +++ b/Telegram/SourceFiles/media/streaming/media_streaming_audio_track.cpp @@ -27,6 +27,7 @@ AudioTrack::AudioTrack( , _ready(std::move(ready)) , _error(std::move(error)) , _playPosition(options.position) { + Expects(_stream.duration > 1); Expects(_ready != nullptr); Expects(_error != nullptr); Expects(_audioId.externalPlayId() != 0); @@ -41,6 +42,10 @@ AVRational AudioTrack::streamTimeBase() const { return _stream.timeBase; } +crl::time AudioTrack::streamDuration() const { + return _stream.duration; +} + void AudioTrack::process(Packet &&packet) { _noMoreData = packet.empty(); if (initialized()) { @@ -193,7 +198,11 @@ rpl::producer AudioTrack::playPosition() { if (state.waitingForData) { _waitingForData.fire({}); } - _playPosition = state.position * 1000 / state.frequency; + _playPosition = std::clamp( + ((state.position * 1000 + (state.frequency / 2)) + / state.frequency), + crl::time(0), + _stream.duration - 1); return; case State::Paused: return; diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_audio_track.h b/Telegram/SourceFiles/media/streaming/media_streaming_audio_track.h index d7453bc24..b93007478 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_audio_track.h +++ b/Telegram/SourceFiles/media/streaming/media_streaming_audio_track.h @@ -40,6 +40,7 @@ public: // Thread-safe. [[nodiscard]] int streamIndex() const; [[nodiscard]] AVRational streamTimeBase() const; + [[nodiscard]] crl::time streamDuration() const; // Called from the same unspecified thread. void process(Packet &&packet); diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_common.h b/Telegram/SourceFiles/media/streaming/media_streaming_common.h index 32f9b7842..0bc7e448d 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_common.h +++ b/Telegram/SourceFiles/media/streaming/media_streaming_common.h @@ -38,6 +38,7 @@ struct PlaybackOptions { AudioMsgId audioId; bool syncVideoByAudio = true; bool dropStaleFrames = true; + bool loop = false; }; struct TrackState { diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_file.cpp b/Telegram/SourceFiles/media/streaming/media_streaming_file.cpp index e38acf98d..8cc0fe7e1 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_file.cpp +++ b/Telegram/SourceFiles/media/streaming/media_streaming_file.cpp @@ -129,11 +129,14 @@ Stream File::Context::initStream(AVMediaType type) { } result.timeBase = info->time_base; result.duration = (info->duration != AV_NOPTS_VALUE) - ? PtsToTimeCeil(info->duration, result.timeBase) - : PtsToTimeCeil(_format->duration, kUniversalTimeBase); + ? PtsToTime(info->duration, result.timeBase) + : PtsToTime(_format->duration, kUniversalTimeBase); if (result.duration == kTimeUnknown || !result.duration) { return {}; } + // We want duration to be greater than any valid frame position. + // That way we can handle looping by advancing position by n * duration. + ++result.duration; return result; } @@ -217,6 +220,9 @@ void File::Context::start(crl::time position) { } _reader->headerDone(); + _totalDuration = std::max( + video.codec ? video.duration : kTimeUnknown, + audio.codec ? audio.duration : kTimeUnknown); if (video.codec || audio.codec) { seekToPosition(video.codec ? video : audio, position); } @@ -224,7 +230,9 @@ void File::Context::start(crl::time position) { return; } - _delegate->fileReady(std::move(video), std::move(audio)); + if (!_delegate->fileReady(std::move(video), std::move(audio))) { + return fail(); + } } void File::Context::readNextPacket() { @@ -248,8 +256,19 @@ void File::Context::readNextPacket() { void File::Context::handleEndOfFile() { const auto more = _delegate->fileProcessPacket(Packet()); - // #TODO streaming later looping - _readTillEnd = true; + if (_delegate->fileReadMore()) { + _readTillEnd = false; + auto error = AvErrorWrap(av_seek_frame( + _format.get(), + -1, // stream_index + 0, // timestamp + AVSEEK_FLAG_BACKWARD)); + if (error) { + logFatal(qstr("av_seek_frame")); + } + } else { + _readTillEnd = true; + } } void File::Context::interrupt() { diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_file.h b/Telegram/SourceFiles/media/streaming/media_streaming_file.h index 6fc71874c..20f5b3a5d 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_file.h +++ b/Telegram/SourceFiles/media/streaming/media_streaming_file.h @@ -88,6 +88,7 @@ private: std::atomic _interrupted = false; FormatPointer _format; + crl::time _totalDuration = kTimeUnknown; }; diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_file_delegate.h b/Telegram/SourceFiles/media/streaming/media_streaming_file_delegate.h index 8cab74568..9eccbc29b 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_file_delegate.h +++ b/Telegram/SourceFiles/media/streaming/media_streaming_file_delegate.h @@ -15,7 +15,9 @@ class Packet; class FileDelegate { public: - virtual void fileReady(Stream &&video, Stream &&audio) = 0; + [[nodiscard]] virtual bool fileReady( + Stream &&video, + Stream &&audio) = 0; virtual void fileError() = 0; virtual void fileWaitingForData() = 0; diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_player.cpp b/Telegram/SourceFiles/media/streaming/media_streaming_player.cpp index 2af53981c..1ed46c0b2 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_player.cpp +++ b/Telegram/SourceFiles/media/streaming/media_streaming_player.cpp @@ -18,7 +18,6 @@ namespace Media { namespace Streaming { namespace { -constexpr auto kReceivedTillEnd = std::numeric_limits::max(); constexpr auto kBufferFor = 3 * crl::time(1000); constexpr auto kLoadInAdvanceFor = 64 * crl::time(1000); constexpr auto kMsFrequency = 1000; // 1000 ms per second. @@ -27,16 +26,6 @@ constexpr auto kMsFrequency = 1000; // 1000 ms per second. // slower than we're playing, so load full file in that case. constexpr auto kLoadFullIfStuckAfterPlayback = 3 * crl::time(1000); -[[nodiscard]] crl::time TrackClampReceivedTill( - crl::time position, - const TrackState &state) { - return (state.duration == kTimeUnknown || position == kTimeUnknown) - ? position - : (position == kReceivedTillEnd) - ? state.duration - : std::clamp(position, crl::time(0), state.duration - 1); -} - [[nodiscard]] bool FullTrackReceived(const TrackState &state) { return (state.duration != kTimeUnknown) && (state.receivedTill == state.duration); @@ -126,7 +115,6 @@ void Player::trackReceivedTill( if (position == kTimeUnknown) { return; } else if (state.duration != kTimeUnknown) { - position = std::clamp(position, crl::time(0), state.duration); if (state.receivedTill < position) { state.receivedTill = position; trackSendReceivedTill(track, state); @@ -150,9 +138,11 @@ void Player::trackPlayedTill( const auto guard = base::make_weak(&_sessionGuard); trackReceivedTill(track, state, position); if (guard && position != kTimeUnknown) { - position = std::clamp(position, crl::time(0), state.duration); state.position = position; - _updates.fire({ PlaybackUpdate{ position } }); + const auto value = _options.loop + ? position + : (position % _totalDuration); + _updates.fire({ PlaybackUpdate{ value } }); } if (_pauseReading && (!bothReceivedEnough(kLoadInAdvanceFor) || receivedTillEnd())) { @@ -169,13 +159,15 @@ void Player::trackSendReceivedTill( Expects(state.duration != kTimeUnknown); Expects(state.receivedTill != kTimeUnknown); - _updates.fire({ PreloadedUpdate{ state.receivedTill } }); + const auto value = _options.loop + ? state.receivedTill + : (state.receivedTill % _totalDuration); + _updates.fire({ PreloadedUpdate{ value } }); } void Player::audioReceivedTill(crl::time position) { Expects(_audio != nullptr); - position = TrackClampReceivedTill(position, _information.audio.state); trackReceivedTill(*_audio, _information.audio.state, position); checkResumeFromWaitingForData(); } @@ -189,8 +181,8 @@ void Player::audioPlayedTill(crl::time position) { void Player::videoReceivedTill(crl::time position) { Expects(_video != nullptr); - position = TrackClampReceivedTill(position, _information.video.state); trackReceivedTill(*_video, _information.video.state, position); + checkResumeFromWaitingForData(); } void Player::videoPlayedTill(crl::time position) { @@ -199,7 +191,7 @@ void Player::videoPlayedTill(crl::time position) { trackPlayedTill(*_video, _information.video.state, position); } -void Player::fileReady(Stream &&video, Stream &&audio) { +bool Player::fileReady(Stream &&video, Stream &&audio) { _waitingForData = false; const auto weak = base::make_weak(&_sessionGuard); @@ -248,8 +240,14 @@ void Player::fileReady(Stream &&video, Stream &&audio) { || (!_audio && !_video)) { LOG(("Streaming Error: Required stream not found for mode %1." ).arg(int(mode))); - fileError(); + return false; } + _totalDuration = std::max( + _audio ? _audio->streamDuration() : kTimeUnknown, + _video ? _video->streamDuration() : kTimeUnknown); + + Ensures(_totalDuration > 1); + return true; } void Player::fileError() { @@ -281,27 +279,35 @@ bool Player::fileProcessPacket(Packet &&packet) { if (packet.empty()) { _readTillEnd = true; if (_audio) { + const auto till = _loopingShift + _audio->streamDuration(); crl::on_main(&_sessionGuard, [=] { - audioReceivedTill(kReceivedTillEnd); + audioReceivedTill(till); }); _audio->process(Packet()); } if (_video) { + const auto till = _loopingShift + _video->streamDuration(); crl::on_main(&_sessionGuard, [=] { - videoReceivedTill(kReceivedTillEnd); + videoReceivedTill(till); }); _video->process(Packet()); } } else if (_audio && _audio->streamIndex() == native.stream_index) { - const auto time = PacketPosition(packet, _audio->streamTimeBase()); + const auto till = _loopingShift + std::clamp( + PacketPosition(packet, _audio->streamTimeBase()), + crl::time(0), + _audio->streamDuration() - 1); crl::on_main(&_sessionGuard, [=] { - audioReceivedTill(time); + audioReceivedTill(till); }); _audio->process(std::move(packet)); } else if (_video && _video->streamIndex() == native.stream_index) { - const auto time = PacketPosition(packet, _video->streamTimeBase()); + const auto till = _loopingShift + std::clamp( + PacketPosition(packet, _video->streamTimeBase()), + crl::time(0), + _video->streamDuration() - 1); crl::on_main(&_sessionGuard, [=] { - videoReceivedTill(time); + videoReceivedTill(till); }); _video->process(std::move(packet)); } @@ -309,7 +315,11 @@ bool Player::fileProcessPacket(Packet &&packet) { } bool Player::fileReadMore() { - // return true if looping. + if (_options.loop && _readTillEnd) { + _readTillEnd = false; + _loopingShift += _totalDuration; + return true; + } return !_readTillEnd && !_pauseReading; } @@ -363,6 +373,9 @@ void Player::fail() { void Player::play(const PlaybackOptions &options) { Expects(options.speed >= 0.5 && options.speed <= 2.); + // Looping video with audio is not supported for now. + Expects(!options.loop || (options.mode != Mode::Both)); + stop(); _lastFailureStage = Stage::Uninitialized; @@ -431,18 +444,22 @@ void Player::updatePausedState() { bool Player::trackReceivedEnough( const TrackState &state, crl::time amount) const { - return FullTrackReceived(state) + return (!_options.loop && FullTrackReceived(state)) || (state.position != kTimeUnknown - && state.position + amount <= state.receivedTill); + && (state.position + std::min(amount, state.duration) + <= state.receivedTill)); } bool Player::bothReceivedEnough(crl::time amount) const { - auto &info = _information; + const auto &info = _information; return (!_audio || trackReceivedEnough(info.audio.state, amount)) && (!_video || trackReceivedEnough(info.video.state, amount)); } bool Player::receivedTillEnd() const { + if (_options.loop) { + return false; + } return (!_video || FullTrackReceived(_information.video.state)) && (!_audio || FullTrackReceived(_information.audio.state)); } @@ -490,6 +507,7 @@ void Player::start() { _video->renderNextFrame( ) | rpl::start_with_next_done([=](crl::time when) { _nextFrameTime = when; + LOG(("RENDERING AT: %1").arg(when)); checkNextFrame(); }, [=] { Expects(_stage == Stage::Started); @@ -521,6 +539,7 @@ void Player::stop() { _videoFinished = false; _pauseReading = false; _readTillEnd = false; + _loopingShift = 0; _information = Information(); } @@ -604,13 +623,15 @@ Media::Player::TrackState Player::prepareLegacyState() const { _information.video.state.position); if (result.position == kTimeUnknown) { result.position = _options.position; + } else if (_options.loop && _totalDuration > 0) { + result.position %= _totalDuration; } - result.length = std::max( - _information.audio.state.duration, - _information.video.state.duration); - if (result.length == kTimeUnknown && _options.audioId.audio()) { + result.length = _totalDuration; + if (result.length == kTimeUnknown) { const auto document = _options.audioId.audio(); - const auto duration = document->song() + const auto duration = !document + ? crl::time(0) + : document->song() ? document->song()->duration : document->duration(); if (duration > 0) { diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_player.h b/Telegram/SourceFiles/media/streaming/media_streaming_player.h index 1720e8134..2471377f6 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_player.h +++ b/Telegram/SourceFiles/media/streaming/media_streaming_player.h @@ -78,7 +78,7 @@ private: not_null delegate(); // FileDelegate methods are called only from the File thread. - void fileReady(Stream &&video, Stream &&audio) override; + bool fileReady(Stream &&video, Stream &&audio) override; void fileError() override; void fileWaitingForData() override; bool fileProcessPacket(Packet &&packet) override; @@ -131,6 +131,9 @@ private: // Immutable while File is active. base::has_weak_ptr _sessionGuard; + + // Immutable while File is active except '.speed'. + // '.speed' is changed from the main thread. PlaybackOptions _options; // Belongs to the File thread while File is active. @@ -149,6 +152,8 @@ private: bool _audioFinished = false; bool _videoFinished = false; + crl::time _totalDuration = 0; + crl::time _loopingShift = 0; crl::time _startedTime = kTimeUnknown; crl::time _pausedTime = kTimeUnknown; crl::time _nextFrameTime = kTimeUnknown; diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_video_track.cpp b/Telegram/SourceFiles/media/streaming/media_streaming_video_track.cpp index fe5f90010..c8a0ddc01 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_video_track.cpp +++ b/Telegram/SourceFiles/media/streaming/media_streaming_video_track.cpp @@ -46,15 +46,23 @@ public: void updateFrameRequest(const FrameRequest &request); private: + enum class FrameResult { + Done, + Error, + Waiting, + Looped, + Finished, + }; [[nodiscard]] bool interrupted() const; [[nodiscard]] bool tryReadFirstFrame(Packet &&packet); [[nodiscard]] bool fillStateFromFrame(); [[nodiscard]] bool processFirstFrame(); void queueReadFrames(crl::time delay = 0); void readFrames(); - [[nodiscard]] bool readFrame(not_null frame); + [[nodiscard]] FrameResult readFrame(not_null frame); void presentFrameIfNeeded(); void callReady(); + void loopAround(); // Force frame position to be clamped to [0, duration] and monotonic. [[nodiscard]] crl::time currentFramePosition() const; @@ -77,6 +85,7 @@ private: crl::time _resumedTime = kTimeUnknown; mutable TimePoint _syncTimePoint; mutable crl::time _previousFramePosition = kTimeUnknown; + crl::time _framePositionShift = 0; crl::time _nextFrameDisplayTime = kTimeUnknown; rpl::event_stream _nextFrameTimeUpdates; rpl::event_stream<> _waitingForData; @@ -103,6 +112,7 @@ VideoTrackObject::VideoTrackObject( , _ready(std::move(ready)) , _error(std::move(error)) , _readFramesTimer(_weak, [=] { readFrames(); }) { + Expects(_stream.duration > 1); Expects(_ready != nullptr); Expects(_error != nullptr); } @@ -151,14 +161,24 @@ void VideoTrackObject::readFrames() { if (interrupted()) { return; } - const auto time = trackTime().trackTime; + auto time = trackTime().trackTime; const auto dropStaleFrames = _options.dropStaleFrames; const auto state = _shared->prepareState(time, dropStaleFrames); state.match([&](Shared::PrepareFrame frame) { - while (readFrame(frame)) { - if (!dropStaleFrames || !VideoTrack::IsStale(frame, time)) { + while (true) { + const auto result = readFrame(frame); + if (result == FrameResult::Looped) { + time -= _stream.duration; + continue; + } else if (result != FrameResult::Done) { + break; + } else if (!dropStaleFrames + || !VideoTrack::IsStale(frame, time)) { + LOG(("READ FRAMES, TRACK TIME: %1").arg(time)); presentFrameIfNeeded(); break; + } else { + LOG(("DROPPED FRAMES, TRACK TIME: %1").arg(time)); } } }, [&](Shared::PrepareNextCheck delay) { @@ -170,29 +190,43 @@ void VideoTrackObject::readFrames() { }); } -bool VideoTrackObject::readFrame(not_null frame) { +void VideoTrackObject::loopAround() { + LOG(("LOOPING AROUND")); + avcodec_flush_buffers(_stream.codec.get()); + _framePositionShift += _stream.duration; +} + +auto VideoTrackObject::readFrame(not_null frame) -> FrameResult { if (const auto error = ReadNextFrame(_stream)) { if (error.code() == AVERROR_EOF) { - interrupt(); - _nextFrameTimeUpdates = rpl::event_stream(); + if (_options.loop) { + loopAround(); + return FrameResult::Looped; + } else { + interrupt(); + _nextFrameTimeUpdates = rpl::event_stream(); + return FrameResult::Finished; + } } else if (error.code() != AVERROR(EAGAIN) || _noMoreData) { interrupt(); _error(); - } else if (_stream.queue.empty()) { - _waitingForData.fire({}); + return FrameResult::Error; } - return false; + Assert(_stream.queue.empty()); + _waitingForData.fire({}); + return FrameResult::Waiting; } const auto position = currentFramePosition(); + LOG(("GOT FRAME: %1 (queue %2)").arg(position).arg(_stream.queue.size())); if (position == kTimeUnknown) { interrupt(); _error(); - return false; + return FrameResult::Error; } std::swap(frame->decoded, _stream.frame); frame->position = position; frame->displayed = kTimeUnknown; - return true; + return FrameResult::Done; } void VideoTrackObject::presentFrameIfNeeded() { @@ -226,6 +260,7 @@ void VideoTrackObject::presentFrameIfNeeded() { // we assign a new value, even if the value really didn't change. _nextFrameDisplayTime = time.worldTime + crl::time(std::round(trackLeft / _options.speed)); + LOG(("NOW: %1, FRAME POSITION: %2, TRACK TIME: %3, TRACK LEFT: %4, NEXT: %5").arg(time.worldTime).arg(presented.displayPosition).arg(time.trackTime).arg(trackLeft).arg(_nextFrameDisplayTime)); _nextFrameTimeUpdates.fire_copy(_nextFrameDisplayTime); } queueReadFrames(presented.nextCheckDelay); @@ -332,9 +367,9 @@ bool VideoTrackObject::processFirstFrame() { } crl::time VideoTrackObject::currentFramePosition() const { - const auto position = std::min( + const auto position = _framePositionShift + std::min( FramePosition(_stream), - _stream.duration); + _stream.duration - 1); if (_previousFramePosition != kTimeUnknown && position <= _previousFramePosition) { return kTimeUnknown; @@ -496,6 +531,7 @@ auto VideoTrack::Shared::presentFrame( return { kTimeUnknown, (trackTime - frame->position + 1) }; }; + LOG(("PRESENT COUNTER: %1").arg(counter())); switch (counter()) { case 0: return present(0, 1); case 1: return nextCheckDelay(2); @@ -548,6 +584,7 @@ VideoTrack::VideoTrack( Fn error) : _streamIndex(stream.index) , _streamTimeBase(stream.timeBase) +, _streamDuration(stream.duration) //, _streamRotation(stream.rotation) , _shared(std::make_unique()) , _wrapped( @@ -567,6 +604,10 @@ AVRational VideoTrack::streamTimeBase() const { return _streamTimeBase; } +crl::time VideoTrack::streamDuration() const { + return _streamDuration; +} + void VideoTrack::process(Packet &&packet) { _wrapped.with([ packet = std::move(packet) diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_video_track.h b/Telegram/SourceFiles/media/streaming/media_streaming_video_track.h index 6973a7da5..5dc415ed1 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_video_track.h +++ b/Telegram/SourceFiles/media/streaming/media_streaming_video_track.h @@ -30,6 +30,7 @@ public: // Thread-safe. [[nodiscard]] int streamIndex() const; [[nodiscard]] AVRational streamTimeBase() const; + [[nodiscard]] crl::time streamDuration() const; // Called from the same unspecified thread. void process(Packet &&packet); @@ -120,6 +121,7 @@ private: const int _streamIndex = 0; const AVRational _streamTimeBase; + const crl::time _streamDuration = 0; //const int _streamRotation = 0; std::unique_ptr _shared; diff --git a/Telegram/SourceFiles/media/view/media_view_overlay_widget.cpp b/Telegram/SourceFiles/media/view/media_view_overlay_widget.cpp index c6dfd745f..36ac74b76 100644 --- a/Telegram/SourceFiles/media/view/media_view_overlay_widget.cpp +++ b/Telegram/SourceFiles/media/view/media_view_overlay_widget.cpp @@ -2187,6 +2187,10 @@ void OverlayWidget::restartAtSeekPosition(crl::time position) { } auto options = Streaming::PlaybackOptions(); options.position = position; + if (_doc->isAnimation() || true) { + options.mode = Streaming::Mode::Video; + options.loop = true; + } _streamed->player.play(options); Media::Player::instance()->pause(AudioMsgId::Type::Voice); @@ -2318,7 +2322,6 @@ void OverlayWidget::paintEvent(QPaintEvent *e) { : rects; auto ms = crl::now(); - const auto guard = gsl::finally([&] { LOG(("FULL FRAME: %1").arg(crl::now() - ms)); }); Painter p(this);