From ccd04b98b9f6592404ae6d48745185d9c41f2c89 Mon Sep 17 00:00:00 2001 From: John Preston Date: Mon, 25 Feb 2019 14:17:25 +0400 Subject: [PATCH] Fix sync video to audio. --- Telegram/SourceFiles/data/data_document.cpp | 2 +- .../streaming/media_streaming_audio_track.cpp | 2 +- .../media/streaming/media_streaming_file.cpp | 12 +++----- .../media_streaming_loader_mtproto.cpp | 4 +-- .../streaming/media_streaming_player.cpp | 9 ------ .../streaming/media_streaming_utility.cpp | 20 ------------- .../streaming/media_streaming_video_track.cpp | 30 +++++++++++-------- .../streaming/media_streaming_video_track.h | 1 - 8 files changed, 26 insertions(+), 54 deletions(-) diff --git a/Telegram/SourceFiles/data/data_document.cpp b/Telegram/SourceFiles/data/data_document.cpp index d174c508e..373155f89 100644 --- a/Telegram/SourceFiles/data/data_document.cpp +++ b/Telegram/SourceFiles/data/data_document.cpp @@ -28,7 +28,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include "mainwindow.h" #include "core/application.h" -// #TODO streaming +// #TODO streaming ui #include "media/streaming/media_streaming_player.h" #include "media/streaming/media_streaming_loader_mtproto.h" diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_audio_track.cpp b/Telegram/SourceFiles/media/streaming/media_streaming_audio_track.cpp index 1b0d51733..ca4730862 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_audio_track.cpp +++ b/Telegram/SourceFiles/media/streaming/media_streaming_audio_track.cpp @@ -181,7 +181,7 @@ rpl::producer AudioTrack::playPosition() { const auto type = AudioMsgId::Type::Video; const auto state = Media::Player::mixer()->currentState(type); if (state.id != _audioId) { - // #TODO streaming muted by other + // #TODO streaming later muted by other return; } else switch (state.state) { case State::Stopped: diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_file.cpp b/Telegram/SourceFiles/media/streaming/media_streaming_file.cpp index ca706b655..34b12d414 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_file.cpp +++ b/Telegram/SourceFiles/media/streaming/media_streaming_file.cpp @@ -10,8 +10,6 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include "media/streaming/media_streaming_loader.h" #include "media/streaming/media_streaming_file_delegate.h" -#include "ui/toast/toast.h" // #TODO streaming - namespace Media { namespace Streaming { @@ -266,9 +264,10 @@ void File::Context::readNextPacket() { handleEndOfFile(); } } + void File::Context::handleEndOfFile() { const auto more = _delegate->fileProcessPacket(Packet()); - // #TODO streaming looping + // #TODO streaming later looping _readTillEnd = true; } @@ -314,7 +313,8 @@ File::Context::~Context() { } bool File::Context::finished() const { - return unroll() || _readTillEnd; // #TODO streaming looping + // #TODO streaming later looping + return unroll() || _readTillEnd; } File::File( @@ -332,10 +332,6 @@ void File::start(not_null delegate, crl::time position) { while (!context->finished()) { context->readNextPacket(); } - - crl::on_main(context, [] { AssertIsDebug(); - Ui::Toast::Show("Finished loading."); - }); }); } diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_loader_mtproto.cpp b/Telegram/SourceFiles/media/streaming/media_streaming_loader_mtproto.cpp index c5bacb8d0..672a6f815 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_loader_mtproto.cpp +++ b/Telegram/SourceFiles/media/streaming/media_streaming_loader_mtproto.cpp @@ -96,7 +96,7 @@ void LoaderMtproto::changeCdnParams( const QByteArray &encryptionKey, const QByteArray &encryptionIV, const QVector &hashes) { - // #TODO streaming + // #TODO streaming cdn } void LoaderMtproto::requestFailed(int offset, const RPCError &error) { @@ -106,7 +106,7 @@ void LoaderMtproto::requestFailed(int offset, const RPCError &error) { return; } const auto callback = [=](const Data::UpdatedFileReferences &updated) { - // #TODO streaming + // #TODO streaming file_reference }; _api->refreshFileReference(_origin, crl::guard(this, callback)); } diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_player.cpp b/Telegram/SourceFiles/media/streaming/media_streaming_player.cpp index 31b8b126e..6722a8071 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_player.cpp +++ b/Telegram/SourceFiles/media/streaming/media_streaming_player.cpp @@ -98,7 +98,6 @@ void Player::checkNextFrame() { if (now < _nextFrameTime) { _renderFrameTimer.callOnce(_nextFrameTime - now); } else { - _renderFrameTimer.cancel(); renderFrame(now); } } @@ -269,20 +268,12 @@ bool Player::fileProcessPacket(Packet &&packet) { } } else if (_audio && _audio->streamIndex() == native.stream_index) { const auto time = PacketPosition(packet, _audio->streamTimeBase()); - //LOG(("[%2] AUDIO PACKET FOR %1ms" - // ).arg(time - // ).arg(crl::now() % 10000, 4, 10, QChar('0'))); - crl::on_main(&_sessionGuard, [=] { audioReceivedTill(time); }); _audio->process(std::move(packet)); } else if (_video && _video->streamIndex() == native.stream_index) { const auto time = PacketPosition(packet, _video->streamTimeBase()); - //LOG(("[%2] VIDEO PACKET FOR %1ms" - // ).arg(time - // ).arg(crl::now() % 10000, 4, 10, QChar('0'))); - crl::on_main(&_sessionGuard, [=] { videoReceivedTill(time); }); diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_utility.cpp b/Telegram/SourceFiles/media/streaming/media_streaming_utility.cpp index f75965bc9..5285ce629 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_utility.cpp +++ b/Telegram/SourceFiles/media/streaming/media_streaming_utility.cpp @@ -343,26 +343,6 @@ QImage ConvertFrame( return QImage(); } } - //if (stream.rotation == 180) { - // storage = std::move(storage).mirrored(true, true); - //} else if (stream.rotation != 0) { - // auto transform = QTransform(); - // transform.rotate(stream.rotation); - // storage = storage.transformed(transform); - //} - - // Read some future packets for audio stream. - //if (_audioStreamId >= 0) { - // while (_frameMs + 5000 > _lastReadAudioMs - // && _frameMs + 15000 > _lastReadVideoMs) { - // auto packetResult = readAndProcessPacket(); - // if (packetResult != PacketResult::Ok) { - // break; - // } - // } - //} - // #TODO streaming - ClearFrameMemory(stream.frame.get()); return storage; } diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_video_track.cpp b/Telegram/SourceFiles/media/streaming/media_streaming_video_track.cpp index 53116b03a..5712b5c7a 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_video_track.cpp +++ b/Telegram/SourceFiles/media/streaming/media_streaming_video_track.cpp @@ -76,7 +76,8 @@ private: crl::time _resumedTime = kTimeUnknown; mutable TimePoint _syncTimePoint; mutable crl::time _previousFramePosition = kTimeUnknown; - rpl::variable _nextFrameDisplayTime = kTimeUnknown; + crl::time _nextFrameDisplayTime = kTimeUnknown; + rpl::event_stream _nextFrameTimeUpdates; rpl::event_stream<> _waitingForData; bool _queued = false; @@ -110,7 +111,10 @@ VideoTrackObject::VideoTrackObject( rpl::producer VideoTrackObject::displayFrameAt() const { return interrupted() ? rpl::complete() - : _nextFrameDisplayTime.value(); + : (_nextFrameDisplayTime == kTimeUnknown) + ? _nextFrameTimeUpdates.events() + : _nextFrameTimeUpdates.events_starting_with_copy( + _nextFrameDisplayTime); } rpl::producer<> VideoTrackObject::waitingForData() const { @@ -164,7 +168,7 @@ bool VideoTrackObject::readFrame(not_null frame) { if (const auto error = ReadNextFrame(_stream)) { if (error.code() == AVERROR_EOF) { interrupt(); - _nextFrameDisplayTime.reset(kTimeUnknown); + _nextFrameTimeUpdates = rpl::event_stream(); } else if (error.code() != AVERROR(EAGAIN) || _noMoreData) { interrupt(); _error(); @@ -184,9 +188,9 @@ bool VideoTrackObject::readFrame(not_null frame) { QSize(), std::move(frame->original)); frame->position = position; - frame->displayPosition = position; // #TODO streaming adjust / sync frame->displayed = kTimeUnknown; + // #TODO streaming later prepare frame //frame->request //frame->prepared @@ -201,8 +205,12 @@ void VideoTrackObject::presentFrameIfNeeded() { const auto presented = _shared->presentFrame(time.trackTime); if (presented.displayPosition != kTimeUnknown) { const auto trackLeft = presented.displayPosition - time.trackTime; + + // We don't use rpl::variable, because we want an event each time + // we assign a new value, even if the value really didn't change. _nextFrameDisplayTime = time.worldTime + crl::time(std::round(trackLeft / _options.speed)); + _nextFrameTimeUpdates.fire_copy(_nextFrameDisplayTime); } queueReadFrames(presented.nextCheckDelay); } @@ -300,7 +308,6 @@ bool VideoTrackObject::processFirstFrame() { return false; } _shared->init(std::move(frame), _syncTimePoint.trackTime); - _nextFrameDisplayTime.reset(_syncTimePoint.trackTime); callReady(); if (!_stream.queue.empty()) { queueReadFrames(); @@ -383,7 +390,6 @@ void VideoTrack::Shared::init(QImage &&cover, crl::time position) { _frames[0].original = std::move(cover); _frames[0].position = position; - _frames[0].displayPosition = position; // Usually main thread sets displayed time before _counter increment. // But in this case we update _counter, so we set a fake displayed time. @@ -407,7 +413,7 @@ not_null VideoTrack::Shared::getFrame(int index) { } bool VideoTrack::Shared::IsPrepared(not_null frame) { - return (frame->displayPosition != kTimeUnknown) + return (frame->position != kTimeUnknown) && (frame->displayed == kTimeUnknown) && !frame->original.isNull(); } @@ -417,7 +423,7 @@ bool VideoTrack::Shared::IsStale( crl::time trackTime) { Expects(IsPrepared(frame)); - return (frame->displayPosition < trackTime); + return (frame->position < trackTime); } auto VideoTrack::Shared::prepareState(crl::time trackTime) -> PrepareState { @@ -433,7 +439,7 @@ auto VideoTrack::Shared::prepareState(crl::time trackTime) -> PrepareState { } else if (!IsPrepared(next)) { return next; } else { - return PrepareNextCheck(frame->displayPosition - trackTime + 1); + return PrepareNextCheck(frame->position - trackTime + 1); } }; const auto finishPrepare = [&](int index) { @@ -459,7 +465,7 @@ auto VideoTrack::Shared::presentFrame(crl::time trackTime) -> PresentFrame { const auto present = [&](int counter, int index) -> PresentFrame { const auto frame = getFrame(index); Assert(IsPrepared(frame)); - const auto position = frame->displayPosition; + const auto position = frame->position; // Release this frame to the main thread for rendering. _counter.store( @@ -475,7 +481,7 @@ auto VideoTrack::Shared::presentFrame(crl::time trackTime) -> PresentFrame { || IsStale(frame, trackTime)) { return { kTimeUnknown, crl::time(0) }; } - return { kTimeUnknown, (trackTime - frame->displayPosition + 1) }; + return { kTimeUnknown, (trackTime - frame->position + 1) }; }; switch (counter()) { @@ -596,7 +602,7 @@ QImage VideoTrack::frame(const FrameRequest &request) const { if (request.resize.isEmpty()) { return frame->original; } else if (frame->prepared.isNull() || frame->request != request) { - // #TODO streaming prepare frame + // #TODO streaming later prepare frame //frame->request = request; //frame->prepared = PrepareFrame( // frame->original, diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_video_track.h b/Telegram/SourceFiles/media/streaming/media_streaming_video_track.h index 37248f9e1..2970888e8 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_video_track.h +++ b/Telegram/SourceFiles/media/streaming/media_streaming_video_track.h @@ -59,7 +59,6 @@ private: struct Frame { QImage original; crl::time position = kTimeUnknown; - crl::time displayPosition = kTimeUnknown; crl::time displayed = kTimeUnknown; FrameRequest request;