From a093cb6274bd8f16b2a326b416be881aac4a2dbf Mon Sep 17 00:00:00 2001 From: John Preston Date: Sun, 17 Feb 2019 10:54:57 +0400 Subject: [PATCH] Move some logic to Media::Streaming::Player. --- Telegram/SourceFiles/data/data_document.cpp | 33 +- Telegram/SourceFiles/mainwindow.cpp | 13 +- .../media/streaming/media_streaming_common.h | 155 +----- .../media/streaming/media_streaming_file.cpp | 450 +++++++----------- .../media/streaming/media_streaming_file.h | 66 +-- .../streaming/media_streaming_file_delegate.h | 29 ++ .../streaming/media_streaming_player.cpp | 123 +++++ .../media/streaming/media_streaming_player.h | 63 +++ .../media/streaming/media_streaming_reader.h | 2 - ...common.cpp => media_streaming_utility.cpp} | 30 +- .../media/streaming/media_streaming_utility.h | 156 ++++++ Telegram/gyp/telegram_sources.txt | 6 +- 12 files changed, 633 insertions(+), 493 deletions(-) create mode 100644 Telegram/SourceFiles/media/streaming/media_streaming_file_delegate.h create mode 100644 Telegram/SourceFiles/media/streaming/media_streaming_player.cpp create mode 100644 Telegram/SourceFiles/media/streaming/media_streaming_player.h rename Telegram/SourceFiles/media/streaming/{media_streaming_common.cpp => media_streaming_utility.cpp} (92%) create mode 100644 Telegram/SourceFiles/media/streaming/media_streaming_utility.h diff --git a/Telegram/SourceFiles/data/data_document.cpp b/Telegram/SourceFiles/data/data_document.cpp index dfad51d69..92171664e 100644 --- a/Telegram/SourceFiles/data/data_document.cpp +++ b/Telegram/SourceFiles/data/data_document.cpp @@ -29,7 +29,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include "core/application.h" // #TODO streaming -#include "media/streaming/media_streaming_file.h" +#include "media/streaming/media_streaming_player.h" #include "media/streaming/media_streaming_loader_mtproto.h" namespace { @@ -307,18 +307,18 @@ void DocumentOpenClickHandler::Open( return; } } - if (playMusic || playVideo) { + if (data->isAudioFile() || data->isVideoFile()) { AssertIsDebug(); if (auto loader = data->createStreamingLoader(origin)) { - static auto file = std::unique_ptr(); - file = std::make_unique( + static auto player = std::unique_ptr(); + player = std::make_unique( &data->owner(), std::move(loader)); data->session().lifetime().add([] { - file = nullptr; + player = nullptr; }); - file->start( - (playMusic + player->init( + (data->isAudioFile() ? Media::Streaming::Mode::Audio : Media::Streaming::Mode::Video), 0); @@ -431,6 +431,25 @@ void DocumentSaveClickHandler::Save( bool forceSavingAs) { if (!data->date) return; + if (data->isAudioFile() || data->isVideoFile()) { + AssertIsDebug(); + if (auto loader = data->createStreamingLoader(origin)) { + static auto player = std::unique_ptr(); + player = std::make_unique( + &data->owner(), + std::move(loader)); + data->session().lifetime().add([] { + player = nullptr; + }); + player->init( + (data->isAudioFile() + ? Media::Streaming::Mode::Audio + : Media::Streaming::Mode::Video), + 0); + } + return; + } + auto filepath = data->filepath( DocumentData::FilePathResolveSaveFromDataSilent, forceSavingAs); diff --git a/Telegram/SourceFiles/mainwindow.cpp b/Telegram/SourceFiles/mainwindow.cpp index 0f3a2335d..c692a62ed 100644 --- a/Telegram/SourceFiles/mainwindow.cpp +++ b/Telegram/SourceFiles/mainwindow.cpp @@ -302,13 +302,18 @@ void MainWindow::destroyLayer() { if (!_layer) { return; } - const auto resetFocus = Ui::InFocusChain(_layer); - if (resetFocus) setFocus(); - _layer = nullptr; + auto layer = base::take(_layer); + const auto resetFocus = Ui::InFocusChain(layer); + if (resetFocus) { + setFocus(); + } + layer = nullptr; if (controller()) { controller()->disableGifPauseReason(Window::GifPauseReason::Layer); } - if (resetFocus) setInnerFocus(); + if (resetFocus) { + setInnerFocus(); + } InvokeQueued(this, [=] { checkHistoryActivation(); }); diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_common.h b/Telegram/SourceFiles/media/streaming/media_streaming_common.h index 6168b30a7..913611d84 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_common.h +++ b/Telegram/SourceFiles/media/streaming/media_streaming_common.h @@ -7,158 +7,49 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL */ #pragma once -extern "C" { -#include -#include -#include -} // extern "C" - namespace Media { namespace Streaming { +constexpr auto kTimeUnknown = crl::time(-1); + enum class Mode { Both, Audio, Video, - Inspection + Inspection, }; struct Information { - static constexpr auto kDurationUnknown = crl::time(-1); + crl::time videoStarted = kTimeUnknown; + crl::time videoDuration = kTimeUnknown; + QSize videoSize; + QImage videoCover; + int videoCoverRotation = 0; - QSize video; - bool audio = false; - crl::time duration = kDurationUnknown; - - crl::time started = 0; - QImage cover; - int rotation = 0; + crl::time audioStarted = kTimeUnknown; + crl::time audioDuration = kTimeUnknown; }; -class AvErrorWrap { -public: - AvErrorWrap(int code = 0) : _code(code) { - } - - [[nodiscard]] explicit operator bool() const { - return (_code < 0); - } - - [[nodiscard]] int code() const { - return _code; - } - - [[nodiscard]] QString text() const { - char string[AV_ERROR_MAX_STRING_SIZE] = { 0 }; - return QString::fromUtf8(av_make_error_string( - string, - sizeof(string), - _code)); - } - -private: - int _code = 0; - +struct RepaintRequest { + crl::time position; }; -class Packet { -public: - Packet() { - setEmpty(); - } - Packet(const AVPacket &data) { - bytes::copy(_data, bytes::object_as_span(&data)); - } - Packet(Packet &&other) { - bytes::copy(_data, other._data); - if (!other.empty()) { - other.release(); - } - } - Packet &operator=(Packet &&other) { - if (this != &other) { - av_packet_unref(&fields()); - bytes::copy(_data, other._data); - if (!other.empty()) { - other.release(); - } - } - return *this; - } - ~Packet() { - av_packet_unref(&fields()); - } - - [[nodiscard]] AVPacket &fields() { - return *reinterpret_cast(_data); - } - [[nodiscard]] const AVPacket &fields() const { - return *reinterpret_cast(_data); - } - - [[nodiscard]] bool empty() const { - return !fields().data; - } - void release() { - setEmpty(); - } - -private: - void setEmpty() { - auto &native = fields(); - av_init_packet(&native); - native.data = nullptr; - native.size = 0; - } - - alignas(alignof(AVPacket)) bytes::type _data[sizeof(AVPacket)]; - +struct WaitingForData { }; -struct CodecDeleter { - void operator()(AVCodecContext *value); -}; -using CodecPointer = std::unique_ptr; -CodecPointer MakeCodecPointer(not_null stream); - -struct FrameDeleter { - void operator()(AVFrame *value); -}; -using FramePointer = std::unique_ptr; -FramePointer MakeFramePointer(); - -struct SwsContextDeleter { - void operator()(SwsContext *value); -}; -using SwsContextPointer = std::unique_ptr; -SwsContextPointer MakeSwsContextPointer( - not_null frame, - QSize resize, - SwsContextPointer *existing = nullptr); - -struct Stream { - CodecPointer codec; - FramePointer frame; - std::deque queue; - crl::time lastReadPositionTime = 0; - int invalidDataPackets = 0; - - // Video only. - int rotation = 0; - SwsContextPointer swsContext; +struct MutedByOther { }; -void LogError(QLatin1String method); -void LogError(QLatin1String method, AvErrorWrap error); +struct Update { + base::variant< + Information, + RepaintRequest, + WaitingForData, + MutedByOther> data; +}; -[[nodiscard]] crl::time PtsToTime(int64_t pts, const AVRational &timeBase); -[[nodiscard]] int ReadRotationFromMetadata(not_null stream); -[[nodiscard]] bool RotationSwapWidthHeight(int rotation); -[[nodiscard]] std::optional ReadNextFrame(Stream &stream); -[[nodiscard]] QImage ConvertFrame( - Stream& stream, - QSize resize, - QImage storage); +struct Error { +}; } // namespace Streaming } // namespace Media diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_file.cpp b/Telegram/SourceFiles/media/streaming/media_streaming_file.cpp index 1b3492809..d98f869c0 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_file.cpp +++ b/Telegram/SourceFiles/media/streaming/media_streaming_file.cpp @@ -8,18 +8,19 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include "media/streaming/media_streaming_file.h" #include "media/streaming/media_streaming_loader.h" +#include "media/streaming/media_streaming_file_delegate.h" -#include "media/audio/media_audio.h" // #TODO streaming -#include "media/audio/media_child_ffmpeg_loader.h" -#include "ui/toast/toast.h" +#include "ui/toast/toast.h" // #TODO streaming namespace Media { namespace Streaming { -File::Context::Context(not_null reader) -: _reader(reader) -, _size(reader->size()) -, _audioMsgId(AudioMsgId::ForVideo()) { +File::Context::Context( + not_null delegate, + not_null reader) +: _delegate(delegate) +, _reader(reader) +, _size(reader->size()) { } int File::Context::Read(void *opaque, uint8_t *buffer, int bufferSize) { @@ -44,7 +45,7 @@ int File::Context::read(bytes::span buffer) { if (_interrupted) { return -1; } else if (_reader->failed()) { - _failed = true; + fail(); return -1; } } @@ -83,70 +84,68 @@ void File::Context::logError(QLatin1String method, AvErrorWrap error) { void File::Context::logFatal(QLatin1String method) { if (!unroll()) { LogError(method); - _failed = true; + fail(); } } void File::Context::logFatal(QLatin1String method, AvErrorWrap error) { if (!unroll()) { LogError(method, error); - _failed = true; + fail(); } } -void File::Context::initStream(StreamWrap &wrap, AVMediaType type) { - wrap.id = av_find_best_stream( +Stream File::Context::initStream(AVMediaType type) { + auto result = Stream(); + const auto index = result.index = av_find_best_stream( _formatContext, type, -1, -1, nullptr, 0); - if (wrap.id < 0) { - return; + if (index < 0) { + return {}; } - wrap.info = _formatContext->streams[wrap.id]; + const auto info = _formatContext->streams[index]; if (type == AVMEDIA_TYPE_VIDEO) { - wrap.stream.rotation = ReadRotationFromMetadata(wrap.info); + result.rotation = ReadRotationFromMetadata(info); + } else if (type == AVMEDIA_TYPE_AUDIO) { + result.frequency = info->codecpar->sample_rate; + if (!result.frequency) { + return {}; + } } - wrap.stream.codec = MakeCodecPointer(wrap.info); - if (!wrap.stream.codec) { - ClearStream(wrap); - return; + result.codec = MakeCodecPointer(info); + if (!result.codec) { + return {}; } - wrap.stream.frame = MakeFramePointer(); - if (!wrap.stream.frame) { - ClearStream(wrap); - return; + result.frame = MakeFramePointer(); + if (!result.frame) { + return {}; } + result.timeBase = info->time_base; + result.duration = (info->duration != AV_NOPTS_VALUE) + ? PtsToTime(info->duration, result.timeBase) + : PtsToTime(_formatContext->duration, kUniversalTimeBase); + return result; } -void File::Context::seekToPosition(crl::time positionTime) { +void File::Context::seekToPosition(crl::time position) { auto error = AvErrorWrap(); - if (!positionTime) { + if (!position) { return; } - const auto &main = mainStream(); - Assert(main.info != nullptr); - const auto timeBase = main.info->time_base; - const auto timeStamp = (positionTime * timeBase.den) - / (1000LL * timeBase.num); + const auto streamIndex = -1; + const auto seekFlags = 0; error = av_seek_frame( _formatContext, - main.id, - timeStamp, - 0); - if (!error) { - return; - } - error = av_seek_frame( - _formatContext, - main.id, - timeStamp, - AVSEEK_FLAG_BACKWARD); + streamIndex, + TimeToPts(position, kUniversalTimeBase), + seekFlags); if (!error) { return; } @@ -168,10 +167,9 @@ base::variant File::Context::readPacket() { return error; } -void File::Context::start(Mode mode, crl::time positionTime) { +void File::Context::start(crl::time position) { auto error = AvErrorWrap(); - _mode = mode; if (unroll()) { return; } @@ -201,221 +199,131 @@ void File::Context::start(Mode mode, crl::time positionTime) { return logFatal(qstr("avformat_find_stream_info"), error); } - initStream(_video, AVMEDIA_TYPE_VIDEO); - initStream(_audio, AVMEDIA_TYPE_AUDIO); - if (!mainStreamUnchecked().info) { - return logFatal(qstr("RequiredStreamAbsent")); - } - - readInformation(positionTime); - - if (_audio.info - && (_mode == Mode::Audio || _mode == Mode::Both)) { // #TODO streaming - Player::mixer()->resume(_audioMsgId, true); - } -} - -auto File::Context::mainStreamUnchecked() const -> const StreamWrap & { - return (_mode == Mode::Video || (_video.info && _mode != Mode::Audio)) - ? _video - : _audio; -} - -auto File::Context::mainStream() const -> const StreamWrap & { - const auto &result = mainStreamUnchecked(); - - Ensures(result.info != nullptr); - return result; -} - -auto File::Context::mainStream() -> StreamWrap & { - return const_cast(((const Context*)this)->mainStream()); -} - -void File::Context::readInformation(crl::time positionTime) { - const auto &main = mainStream(); - const auto info = main.info; - auto information = Information(); - information.duration = PtsToTime(info->duration, info->time_base); - - auto result = readPacket(); - const auto packet = base::get_if(&result); + auto video = initStream(AVMEDIA_TYPE_VIDEO); if (unroll()) { return; - } else if (packet) { - if (positionTime > 0) { - const auto time = CountPacketPositionTime( - _formatContext->streams[packet->fields().stream_index], - *packet); - information.started = (time == Information::kDurationUnknown) - ? positionTime - : time; - } - } else { - information.started = positionTime; } - if (_audio.info - && (_mode == Mode::Audio || _mode == Mode::Both)) { // #TODO streaming - auto soundData = std::make_unique(); - soundData->context = _audio.stream.codec.release(); - soundData->frequency = _audio.info->codecpar->sample_rate; - if (_audio.info->duration == AV_NOPTS_VALUE) { - soundData->length = (_formatContext->duration * soundData->frequency) / AV_TIME_BASE; - } else { - soundData->length = (_audio.info->duration * soundData->frequency * _audio.info->time_base.num) / _audio.info->time_base.den; - } - Player::mixer()->play(_audioMsgId, std::move(soundData), information.started); - } - - if (packet) { - processPacket(std::move(*packet)); - } else { - enqueueEofPackets(); - } - - information.cover = readFirstVideoFrame(); + auto audio = initStream(AVMEDIA_TYPE_AUDIO); if (unroll()) { return; - } else if (!information.cover.isNull()) { - information.video = information.cover.size(); - information.rotation = _video.stream.rotation; - if (RotationSwapWidthHeight(information.rotation)) { - information.video.transpose(); - } } - information.audio = (_audio.info != nullptr); - _information = std::move(information); -} - -QImage File::Context::readFirstVideoFrame() { - auto result = QImage(); - while (_video.info && result.isNull()) { - auto frame = tryReadFirstVideoFrame(); - if (unroll()) { - return QImage(); - } - frame.match([&](QImage &image) { - if (!image.isNull()) { - result = std::move(image); - } else { - ClearStream(_video); - } - }, [&](const AvErrorWrap &error) { - if (error.code() == AVERROR(EAGAIN)) { - readNextPacket(); - } else { - ClearStream(_video); - } - }); - } - if (!_video.info && _mode == Mode::Video) { - logFatal(qstr("RequiredStreamEmpty")); - return QImage(); - } - return result; -} - -base::variant File::Context::tryReadFirstVideoFrame() { - Expects(_video.info != nullptr); - + seekToPosition(position); if (unroll()) { - return AvErrorWrap(); + return; } - const auto error = ReadNextFrame(_video.stream); - if (error) { - if (error->code() == AVERROR_EOF) { - // No valid video stream. - if (_mode == Mode::Video) { - logFatal(qstr("RequiredStreamEmpty")); - } - return QImage(); - } else if (error->code() != AVERROR(EAGAIN)) { - _failed = true; - } - return *error; - } - return ConvertFrame(_video.stream, QSize(), QImage()); + + _delegate->fileReady(std::move(video), std::move(audio)); } -void File::Context::enqueueEofPackets() { - if (_audio.info) { - Enqueue(_audio, Packet()); - } - if (_video.info) { - Enqueue(_video, Packet()); - } - _readTillEnd = true; -} - -void File::Context::processPacket(Packet &&packet) { - const auto &native = packet.fields(); - const auto streamId = native.stream_index; - const auto check = [&](StreamWrap &wrap) { - if ((native.stream_index == wrap.id) && wrap.info) { - // #TODO streaming queue packet to audio player - if ((_mode == Mode::Audio || _mode == Mode::Both) - && (wrap.info == _audio.info)) { - Player::mixer()->feedFromVideo({ &native, _audioMsgId }); - packet.release(); - } else { - Enqueue(wrap, std::move(packet)); - } - return true; - } - return false; - }; - - check(_audio) || check(_video); -} +//void File::Context::readInformation(crl::time position) { +// auto information = Information(); +// auto result = readPacket(); +// const auto packet = base::get_if(&result); +// if (unroll()) { +// return; +// } else if (packet) { +// if (position > 0) { +// const auto time = CountPacketPosition( +// _formatContext->streams[packet->fields().stream_index], +// *packet); +// information.started = (time == Information::kDurationUnknown) +// ? position +// : time; +// } +// } else { +// information.started = position; +// } +// +// if (packet) { +// processPacket(std::move(*packet)); +// } else { +// enqueueEofPackets(); +// } +// +// information.cover = readFirstVideoFrame(); +// if (unroll()) { +// return; +// } else if (!information.cover.isNull()) { +// information.video = information.cover.size(); +// information.rotation = _video.stream.rotation; +// if (RotationSwapWidthHeight(information.rotation)) { +// information.video.transpose(); +// } +// } +// +// information.audio = (_audio.info != nullptr); +// _information = std::move(information); +//} +// +//QImage File::Context::readFirstVideoFrame() { +// auto result = QImage(); +// while (_video.info && result.isNull()) { +// auto frame = tryReadFirstVideoFrame(); +// if (unroll()) { +// return QImage(); +// } +// frame.match([&](QImage &image) { +// if (!image.isNull()) { +// result = std::move(image); +// } else { +// _video = StreamWrap(); +// } +// }, [&](const AvErrorWrap &error) { +// if (error.code() == AVERROR(EAGAIN)) { +// readNextPacket(); +// } else { +// _video = StreamWrap(); +// } +// }); +// } +// if (!_video.info && _mode == Mode::Video) { +// logFatal(qstr("RequiredStreamEmpty")); +// return QImage(); +// } +// return result; +//} +// +//base::variant File::Context::tryReadFirstVideoFrame() { +// Expects(_video.info != nullptr); +// +// if (unroll()) { +// return AvErrorWrap(); +// } +// const auto error = ReadNextFrame(_video.stream); +// if (error) { +// if (error->code() == AVERROR_EOF) { +// // No valid video stream. +// if (_mode == Mode::Video) { +// logFatal(qstr("RequiredStreamEmpty")); +// } +// return QImage(); +// } else if (error->code() != AVERROR(EAGAIN)) { +// fail(); +// } +// return *error; +// } +// return ConvertFrame(_video.stream, QSize(), QImage()); +//} void File::Context::readNextPacket() { auto result = readPacket(); - const auto packet = base::get_if(&result); if (unroll()) { return; - } else if (packet) { - processPacket(std::move(*packet)); + } else if (const auto packet = base::get_if(&result)) { + const auto more = _delegate->fileProcessPacket(std::move(*packet)); } else { // Still trying to read by drain. Assert(result.is()); Assert(result.get().code() == AVERROR_EOF); - enqueueEofPackets(); + handleEndOfFile(); } } - -crl::time File::Context::CountPacketPositionTime( - not_null info, - const Packet &packet) { - const auto &native = packet.fields(); - const auto packetPts = (native.pts == AV_NOPTS_VALUE) - ? native.dts - : native.pts; - const auto &timeBase = info->time_base; - return PtsToTime(packetPts, info->time_base); -} - -void File::Context::ClearStream(StreamWrap &wrap) { - wrap.id = -1; - wrap.stream = Stream(); - wrap.info = nullptr; -} - -crl::time File::Context::CountPacketPositionTime( - const StreamWrap &wrap, - const Packet &packet) { - return CountPacketPositionTime(wrap.info, packet); -} - -void File::Context::Enqueue(StreamWrap &wrap, Packet &&packet) { - const auto time = CountPacketPositionTime(wrap, packet); - if (time != Information::kDurationUnknown) { - wrap.stream.lastReadPositionTime = time; - } - - QMutexLocker lock(&wrap.mutex); - wrap.stream.queue.push_back(std::move(packet)); +void File::Context::handleEndOfFile() { + // #TODO streaming looping + const auto more = _delegate->fileProcessPacket(Packet()); + _readTillEnd = true; } void File::Context::interrupt() { @@ -435,10 +343,12 @@ bool File::Context::unroll() const { return failed() || interrupted(); } -File::Context::~Context() { - ClearStream(_audio); - ClearStream(_video); +void File::Context::fail() { + _failed = true; + _delegate->fileError(); +} +File::Context::~Context() { if (_opened) { avformat_close_input(&_formatContext); } @@ -453,18 +363,8 @@ File::Context::~Context() { } } -bool File::Context::started() const { - return _information.has_value(); -} - bool File::Context::finished() const { - return unroll() || _readTillEnd; -} - -const Media::Streaming::Information & File::Context::information() const { - Expects(_information.has_value()); - - return *_information; + return unroll() || _readTillEnd; // #TODO streaming looping } File::File( @@ -473,54 +373,32 @@ File::File( : _reader(owner, std::move(loader)) { } -void File::start(Mode mode, crl::time positionTime) { - finish(); +void File::start(not_null delegate, crl::time position) { + stop(); - _context = std::make_unique(&_reader); - _thread = std::thread([=, context = _context.get()] { - context->start(mode, positionTime); - if (context->interrupted()) { - return; - } else if (context->failed()) { - crl::on_main(context, [=] { - // #TODO streaming failed - }); - } else { - crl::on_main(context, [=, info = context->information()] { - // #TODO streaming started - }); - while (!context->finished()) { - context->readNextPacket(); - } - crl::on_main(context, [] { AssertIsDebug(); - Ui::Toast::Show("Finished loading."); - }); + _context.emplace(delegate, &_reader); + _thread = std::thread([=, context = &_context.value()] { + context->start(position); + while (!context->finished()) { + context->readNextPacket(); } + + crl::on_main(context, [] { AssertIsDebug(); + Ui::Toast::Show("Finished loading."); + }); }); } -//rpl::producer File::information() const { -// -//} -// -//rpl::producer File::video() const { -// -//} -// -//rpl::producer File::audio() const { -// -//} - -void File::finish() { +void File::stop() { if (_thread.joinable()) { _context->interrupt(); _thread.join(); } - _context = nullptr; + _context.reset(); } File::~File() { - finish(); + stop(); } } // namespace Streaming diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_file.h b/Telegram/SourceFiles/media/streaming/media_streaming_file.h index 75a100ea5..c75cc8dc8 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_file.h +++ b/Telegram/SourceFiles/media/streaming/media_streaming_file.h @@ -8,6 +8,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #pragma once #include "media/streaming/media_streaming_common.h" +#include "media/streaming/media_streaming_utility.h" #include "media/streaming/media_streaming_reader.h" #include "base/bytes.h" #include "base/weak_ptr.h" @@ -22,6 +23,7 @@ namespace Media { namespace Streaming { class Loader; +class FileDelegate; class File final { public: @@ -30,41 +32,28 @@ public: File(const File &other) = delete; File &operator=(const File &other) = delete; - void start(Mode mode, crl::time positionTime); - - //rpl::producer information() const; - //rpl::producer video() const; - //rpl::producer audio() const; + void start(not_null delegate, crl::time position); + void wake(); + void stop(); ~File(); private: - void finish(); - class Context final : public base::has_weak_ptr { public: - Context(not_null reader); + Context(not_null delegate, not_null reader); - void start(Mode mode, crl::time positionTime); + void start(crl::time position); void readNextPacket(); void interrupt(); [[nodiscard]] bool interrupted() const; [[nodiscard]] bool failed() const; - [[nodiscard]] bool started() const; [[nodiscard]] bool finished() const; - [[nodiscard]] const Information &information() const; ~Context(); private: - struct StreamWrap { - int id = -1; - AVStream *info = nullptr; - Stream stream; - QMutex mutex; - }; - static int Read(void *opaque, uint8_t *buffer, int bufferSize); static int64_t Seek(void *opaque, int64_t offset, int whence); @@ -76,36 +65,19 @@ private: void logError(QLatin1String method, AvErrorWrap error); void logFatal(QLatin1String method); void logFatal(QLatin1String method, AvErrorWrap error); + void fail(); - void initStream(StreamWrap &wrap, AVMediaType type); - void seekToPosition(crl::time positionTime); + Stream initStream(AVMediaType type); + void seekToPosition(crl::time position); - // #TODO base::expected. + // TODO base::expected. [[nodiscard]] base::variant readPacket(); - void processPacket(Packet &&packet); - [[nodiscard]] const StreamWrap &mainStreamUnchecked() const; - [[nodiscard]] const StreamWrap &mainStream() const; - [[nodiscard]] StreamWrap &mainStream(); - void readInformation(crl::time positionTime); + void handleEndOfFile(); - [[nodiscard]] QImage readFirstVideoFrame(); - [[nodiscard]] auto tryReadFirstVideoFrame() - -> base::variant; + const not_null _delegate; + const not_null _reader; - void enqueueEofPackets(); - - static void ClearStream(StreamWrap &wrap); - [[nodiscard]] static crl::time CountPacketPositionTime( - not_null info, - const Packet &packet); - [[nodiscard]] static crl::time CountPacketPositionTime( - const StreamWrap &wrap, - const Packet &packet); - static void Enqueue(StreamWrap &wrap, Packet &&packet); - - not_null _reader; - Mode _mode = Mode::Both; int _offset = 0; int _size = 0; bool _failed = false; @@ -113,22 +85,16 @@ private: bool _readTillEnd = false; crl::semaphore _semaphore; std::atomic _interrupted = false; - std::optional _information; uchar *_ioBuffer = nullptr; AVIOContext *_ioContext = nullptr; AVFormatContext *_formatContext = nullptr; - StreamWrap _video; - StreamWrap _audio; - - AudioMsgId _audioMsgId; - }; - std::thread _thread; + std::optional _context; Reader _reader; - std::unique_ptr _context; + std::thread _thread; }; diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_file_delegate.h b/Telegram/SourceFiles/media/streaming/media_streaming_file_delegate.h new file mode 100644 index 000000000..a3072a576 --- /dev/null +++ b/Telegram/SourceFiles/media/streaming/media_streaming_file_delegate.h @@ -0,0 +1,29 @@ +/* +This file is part of Telegram Desktop, +the official desktop application for the Telegram messaging service. + +For license and copyright information please follow this link: +https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL +*/ +#pragma once + +namespace Media { +namespace Streaming { + +struct Stream; +class Packet; + +class FileDelegate { +public: + virtual void fileReady(Stream &&video, Stream &&audio) = 0; + virtual void fileError() = 0; + + // Return true if reading and processing more packets is desired. + // Return false if sleeping until 'wake()' is called is desired. + // Return true after the EOF packet if looping is desired. + [[nodiscard]] virtual bool fileProcessPacket(Packet &&packet) = 0; + [[nodiscard]] virtual bool fileReadMore() = 0; +}; + +} // namespace Streaming +} // namespace Media diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_player.cpp b/Telegram/SourceFiles/media/streaming/media_streaming_player.cpp new file mode 100644 index 000000000..876931812 --- /dev/null +++ b/Telegram/SourceFiles/media/streaming/media_streaming_player.cpp @@ -0,0 +1,123 @@ +/* +This file is part of Telegram Desktop, +the official desktop application for the Telegram messaging service. + +For license and copyright information please follow this link: +https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL +*/ +#include "media/streaming/media_streaming_player.h" + +#include "media/streaming/media_streaming_file.h" +#include "media/streaming/media_streaming_loader.h" +#include "media/audio/media_audio.h" +#include "media/audio/media_child_ffmpeg_loader.h" + +namespace Media { +namespace Streaming { + +crl::time CountPacketPosition( + not_null info, + const Packet &packet) { + const auto &native = packet.fields(); + const auto packetPts = (native.pts == AV_NOPTS_VALUE) + ? native.dts + : native.pts; + const auto & timeBase = info->time_base; + return PtsToTime(packetPts, info->time_base); +} + +// #TODO streaming +//void Enqueue(StreamWrap &wrap, Packet && packet) { +// const auto time = CountPacketPosition(wrap, packet); +// if (time != kTimeUnknown) { +// wrap.stream.lastReadPosition = time; +// } +// wrap.stream.queue.push_back(std::move(packet)); +//} + +Player::Player( + not_null owner, + std::unique_ptr loader) +: _file(std::make_unique(owner, std::move(loader))) { +} + +not_null Player::delegate() { + return static_cast(this); +} + +void Player::fileReady(Stream &&video, Stream &&audio) { + _audio = std::move(audio); + if (_audio.codec && (_mode == Mode::Audio || _mode == Mode::Both)) { + _audioMsgId = AudioMsgId::ForVideo(); + } else { + _audioMsgId = AudioMsgId(); + } +} + +void Player::fileError() { + +} + +bool Player::fileProcessPacket(Packet &&packet) { + const auto &native = packet.fields(); + if (packet.empty()) { + _readTillEnd = true; + } else if (native.stream_index == _audio.index) { + if (_audioMsgId.playId()) { + if (_audio.codec) { + const auto position = PtsToTime(native.pts, _audio.timeBase); + + auto data = std::make_unique(); + data->context = _audio.codec.release(); + data->frequency = _audio.frequency; + data->length = (_audio.duration * data->frequency) / 1000LL; + Media::Player::mixer()->play(_audioMsgId, std::move(data), position); + + // #TODO streaming resume when started playing + Media::Player::mixer()->resume(_audioMsgId, true); + } + Media::Player::mixer()->feedFromVideo({ &native, _audioMsgId }); + packet.release(); + } + //const auto position = PtsToTime(native.pts, stream->timeBase); + } + return fileReadMore(); +} + +bool Player::fileReadMore() { + // return true if looping. + return !_readTillEnd; +} + +void Player::init(Mode mode, crl::time position) { + stop(); + + _mode = mode; + _file->start(delegate(), position); +} + +void Player::pause() { + +} + +void Player::resume() { + +} + +void Player::stop() { + _file->stop(); + _updates = rpl::event_stream(); +} + +bool Player::playing() const { + return false; +} + +rpl::producer Player::updates() const { + return _updates.events(); +} + +Player::~Player() = default; + +} // namespace Streaming +} // namespace Media diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_player.h b/Telegram/SourceFiles/media/streaming/media_streaming_player.h new file mode 100644 index 000000000..77c1e6f9f --- /dev/null +++ b/Telegram/SourceFiles/media/streaming/media_streaming_player.h @@ -0,0 +1,63 @@ +/* +This file is part of Telegram Desktop, +the official desktop application for the Telegram messaging service. + +For license and copyright information please follow this link: +https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL +*/ +#pragma once + +#include "media/streaming/media_streaming_common.h" +#include "media/streaming/media_streaming_file_delegate.h" + +// #TODO streaming move _audio away +#include "media/streaming/media_streaming_utility.h" + +namespace Data { +class Session; +} // namespace Data + +namespace Media { +namespace Streaming { + +class Loader; +class File; + +class Player final : private FileDelegate { +public: + Player(not_null owner, std::unique_ptr loader); + + void init(Mode mode, crl::time position); + void pause(); + void resume(); + void stop(); + + bool playing() const; + + rpl::producer updates() const; + + ~Player(); + +private: + not_null delegate(); + + void fileReady(Stream &&video, Stream &&audio) override; + void fileError() override; + + bool fileProcessPacket(Packet &&packet) override; + bool fileReadMore() override; + + const std::unique_ptr _file; + bool _readTillEnd = false; + + Mode _mode = Mode::Both; + + Stream _audio; + AudioMsgId _audioMsgId; + + rpl::event_stream _updates; + +}; + +} // namespace Streaming +} // namespace Media diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_reader.h b/Telegram/SourceFiles/media/streaming/media_streaming_reader.h index e54e6bf04..dca3d54b3 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_reader.h +++ b/Telegram/SourceFiles/media/streaming/media_streaming_reader.h @@ -23,8 +23,6 @@ class Reader final { public: Reader(not_null owner, std::unique_ptr loader); - static constexpr auto kPartSize = 128 * 1024; - int size() const; bool fill( bytes::span buffer, diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_common.cpp b/Telegram/SourceFiles/media/streaming/media_streaming_utility.cpp similarity index 92% rename from Telegram/SourceFiles/media/streaming/media_streaming_common.cpp rename to Telegram/SourceFiles/media/streaming/media_streaming_utility.cpp index 3e58152e6..af5057aae 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_common.cpp +++ b/Telegram/SourceFiles/media/streaming/media_streaming_utility.cpp @@ -5,6 +5,8 @@ the official desktop application for the Telegram messaging service. For license and copyright information please follow this link: https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL */ +#include "media/streaming/media_streaming_utility.h" + #include "media/streaming/media_streaming_common.h" extern "C" { @@ -143,12 +145,18 @@ void LogError(QLatin1String method, AvErrorWrap error) { ).arg(error.text())); } -crl::time PtsToTime(int64_t pts, const AVRational &timeBase) { - return (pts == AV_NOPTS_VALUE) - ? Information::kDurationUnknown +crl::time PtsToTime(int64_t pts, AVRational timeBase) { + return (pts == AV_NOPTS_VALUE || !timeBase.den) + ? kTimeUnknown : ((pts * 1000LL * timeBase.num) / timeBase.den); } +int64_t TimeToPts(crl::time time, AVRational timeBase) { + return (time == kTimeUnknown || !timeBase.num) + ? AV_NOPTS_VALUE + : (time * timeBase.den) / (1000LL * timeBase.num); +} + int ReadRotationFromMetadata(not_null stream) { const auto tag = av_dict_get(stream->metadata, "rotate", nullptr, 0); if (tag && *tag->value) { @@ -291,13 +299,13 @@ QImage ConvertFrame( return QImage(); } } - if (stream.rotation == 180) { - storage = std::move(storage).mirrored(true, true); - } else if (stream.rotation != 0) { - auto transform = QTransform(); - transform.rotate(stream.rotation); - storage = storage.transformed(transform); - } + //if (stream.rotation == 180) { + // storage = std::move(storage).mirrored(true, true); + //} else if (stream.rotation != 0) { + // auto transform = QTransform(); + // transform.rotate(stream.rotation); + // storage = storage.transformed(transform); + //} // Read some future packets for audio stream. //if (_audioStreamId >= 0) { @@ -312,7 +320,7 @@ QImage ConvertFrame( // #TODO streaming ClearFrameMemory(stream.frame.get()); - return std::move(storage); + return storage; } } // namespace Streaming diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_utility.h b/Telegram/SourceFiles/media/streaming/media_streaming_utility.h new file mode 100644 index 000000000..a1a90d540 --- /dev/null +++ b/Telegram/SourceFiles/media/streaming/media_streaming_utility.h @@ -0,0 +1,156 @@ +/* +This file is part of Telegram Desktop, +the official desktop application for the Telegram messaging service. + +For license and copyright information please follow this link: +https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL +*/ +#pragma once + +#include "media/streaming/media_streaming_common.h" + +extern "C" { +#include +#include +#include +} // extern "C" + +namespace Media { +namespace Streaming { + +constexpr auto kUniversalTimeBase = AVRational{ 1, AV_TIME_BASE }; + +class AvErrorWrap { +public: + AvErrorWrap(int code = 0) : _code(code) { + } + + [[nodiscard]] explicit operator bool() const { + return (_code < 0); + } + + [[nodiscard]] int code() const { + return _code; + } + + [[nodiscard]] QString text() const { + char string[AV_ERROR_MAX_STRING_SIZE] = { 0 }; + return QString::fromUtf8(av_make_error_string( + string, + sizeof(string), + _code)); + } + +private: + int _code = 0; + +}; + +class Packet { +public: + Packet() { + setEmpty(); + } + Packet(const AVPacket &data) { + bytes::copy(_data, bytes::object_as_span(&data)); + } + Packet(Packet &&other) { + bytes::copy(_data, other._data); + if (!other.empty()) { + other.release(); + } + } + Packet &operator=(Packet &&other) { + if (this != &other) { + av_packet_unref(&fields()); + bytes::copy(_data, other._data); + if (!other.empty()) { + other.release(); + } + } + return *this; + } + ~Packet() { + av_packet_unref(&fields()); + } + + [[nodiscard]] AVPacket &fields() { + return *reinterpret_cast(_data); + } + [[nodiscard]] const AVPacket &fields() const { + return *reinterpret_cast(_data); + } + + [[nodiscard]] bool empty() const { + return !fields().data; + } + void release() { + setEmpty(); + } + +private: + void setEmpty() { + auto &native = fields(); + av_init_packet(&native); + native.data = nullptr; + native.size = 0; + } + + alignas(alignof(AVPacket)) bytes::type _data[sizeof(AVPacket)]; + +}; + +struct CodecDeleter { + void operator()(AVCodecContext *value); +}; +using CodecPointer = std::unique_ptr; +CodecPointer MakeCodecPointer(not_null stream); + +struct FrameDeleter { + void operator()(AVFrame *value); +}; +using FramePointer = std::unique_ptr; +FramePointer MakeFramePointer(); + +struct SwsContextDeleter { + void operator()(SwsContext *value); +}; +using SwsContextPointer = std::unique_ptr; +SwsContextPointer MakeSwsContextPointer( + not_null frame, + QSize resize, + SwsContextPointer *existing = nullptr); + +struct Stream { + int index = -1; + crl::time duration = kTimeUnknown; + AVRational timeBase = kUniversalTimeBase; + CodecPointer codec; + FramePointer frame; + std::deque queue; + crl::time lastReadPosition = 0; + int invalidDataPackets = 0; + + // Audio only. + int frequency = 0; + + // Video only. + int rotation = 0; + SwsContextPointer swsContext; +}; + +void LogError(QLatin1String method); +void LogError(QLatin1String method, AvErrorWrap error); + +[[nodiscard]] crl::time PtsToTime(int64_t pts, AVRational timeBase); +[[nodiscard]] int64_t TimeToPts(int64_t pts, AVRational timeBase); +[[nodiscard]] int ReadRotationFromMetadata(not_null stream); +[[nodiscard]] bool RotationSwapWidthHeight(int rotation); +[[nodiscard]] std::optional ReadNextFrame(Stream &stream); +[[nodiscard]] QImage ConvertFrame( + Stream& stream, + QSize resize, + QImage storage); + +} // namespace Streaming +} // namespace Media diff --git a/Telegram/gyp/telegram_sources.txt b/Telegram/gyp/telegram_sources.txt index 7e4181f96..a16679957 100644 --- a/Telegram/gyp/telegram_sources.txt +++ b/Telegram/gyp/telegram_sources.txt @@ -454,16 +454,20 @@ <(src_loc)/media/player/media_player_volume_controller.h <(src_loc)/media/player/media_player_widget.cpp <(src_loc)/media/player/media_player_widget.h -<(src_loc)/media/streaming/media_streaming_common.cpp <(src_loc)/media/streaming/media_streaming_common.h <(src_loc)/media/streaming/media_streaming_file.cpp <(src_loc)/media/streaming/media_streaming_file.h +<(src_loc)/media/streaming/media_streaming_file_delegate.h <(src_loc)/media/streaming/media_streaming_loader.cpp <(src_loc)/media/streaming/media_streaming_loader.h <(src_loc)/media/streaming/media_streaming_loader_mtproto.cpp <(src_loc)/media/streaming/media_streaming_loader_mtproto.h +<(src_loc)/media/streaming/media_streaming_player.cpp +<(src_loc)/media/streaming/media_streaming_player.h <(src_loc)/media/streaming/media_streaming_reader.cpp <(src_loc)/media/streaming/media_streaming_reader.h +<(src_loc)/media/streaming/media_streaming_utility.cpp +<(src_loc)/media/streaming/media_streaming_utility.h <(src_loc)/media/view/media_clip_controller.cpp <(src_loc)/media/view/media_clip_controller.h <(src_loc)/media/view/media_clip_playback.cpp