From 64f2f330f6419d8bd105100f2f58c618694be826 Mon Sep 17 00:00:00 2001 From: John Preston Date: Sat, 16 Feb 2019 14:29:55 +0400 Subject: [PATCH] Render first frame when starting streaming. --- .../media/clip/media_clip_reader.cpp | 7 +- .../streaming/media_streaming_common.cpp | 386 ++++++++++++------ .../media/streaming/media_streaming_common.h | 62 +-- .../media/streaming/media_streaming_file.cpp | 33 +- 4 files changed, 313 insertions(+), 175 deletions(-) diff --git a/Telegram/SourceFiles/media/clip/media_clip_reader.cpp b/Telegram/SourceFiles/media/clip/media_clip_reader.cpp index bd3ee9e03..912f74bb0 100644 --- a/Telegram/SourceFiles/media/clip/media_clip_reader.cpp +++ b/Telegram/SourceFiles/media/clip/media_clip_reader.cpp @@ -369,7 +369,7 @@ public: if (!_implementation && !init()) { return error(); } - if (frame() && frame()->original.isNull()) { + if (frame()->original.isNull()) { auto readResult = _implementation->readFramesTill(-1, ms); if (readResult == internal::ReaderImplementation::ReadResult::EndOfFile && _seekPositionMs > 0) { // If seek was done to the end: try to read the first frame, @@ -460,7 +460,8 @@ public: } bool renderFrame() { - Assert(frame() != 0 && _request.valid()); + Expects(_request.valid()); + if (!_implementation->renderFrame(frame()->original, frame()->alpha, QSize(_request.framew, _request.frameh))) { return false; } @@ -573,7 +574,7 @@ private: }; Frame _frames[3]; int _frame = 0; - Frame *frame() { + not_null frame() { return _frames + _frame; } diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_common.cpp b/Telegram/SourceFiles/media/streaming/media_streaming_common.cpp index 9f0c292ca..3e58152e6 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_common.cpp +++ b/Telegram/SourceFiles/media/streaming/media_streaming_common.cpp @@ -15,109 +15,58 @@ namespace Media { namespace Streaming { namespace { -constexpr int kSkipInvalidDataPackets = 10; +constexpr auto kSkipInvalidDataPackets = 10; +constexpr auto kAlignImageBy = 16; +constexpr auto kPixelBytesSize = 4; + +void AlignedImageBufferCleanupHandler(void* data) { + const auto buffer = static_cast(data); + delete[] buffer; +} + +// Create a QImage of desired size where all the data is properly aligned. +QImage CreateAlignedImage(QSize size) { + const auto width = size.width(); + const auto height = size.height(); + const auto widthAlign = kAlignImageBy / kPixelBytesSize; + const auto neededWidth = width + ((width % widthAlign) + ? (widthAlign - (width % widthAlign)) + : 0); + const auto perLine = neededWidth * kPixelBytesSize; + const auto buffer = new uchar[perLine * height + kAlignImageBy]; + const auto cleanupData = static_cast(buffer); + const auto address = reinterpret_cast(buffer); + const auto alignedBuffer = buffer + ((address % kAlignImageBy) + ? (kAlignImageBy - (address % kAlignImageBy)) + : 0); + return QImage( + alignedBuffer, + width, + height, + perLine, + QImage::Format_ARGB32_Premultiplied, + AlignedImageBufferCleanupHandler, + cleanupData); +} + +bool IsAlignedImage(const QImage &image) { + return !(reinterpret_cast(image.bits()) % kAlignImageBy) + && !(image.bytesPerLine() % kAlignImageBy); +} + +void ClearFrameMemory(AVFrame *frame) { + if (frame && frame->data[0]) { + av_frame_unref(frame); + } +} } // namespace -void LogError(QLatin1String method) { - LOG(("Streaming Error: Error in %1.").arg(method)); -} - -void LogError(QLatin1String method, AvErrorWrap error) { - LOG(("Streaming Error: Error in %1 (code: %2, text: %3)." - ).arg(method - ).arg(error.code() - ).arg(error.text())); -} - -crl::time PtsToTime(int64_t pts, const AVRational &timeBase) { - return (pts == AV_NOPTS_VALUE) - ? Information::kDurationUnknown - : ((pts * 1000LL * timeBase.num) / timeBase.den); -} - -std::optional ReadNextFrame(Stream &stream) { - Expects(stream.frame != nullptr); - +CodecPointer MakeCodecPointer(not_null stream) { auto error = AvErrorWrap(); - if (stream.frame->data) { - av_frame_unref(stream.frame.get()); - } - do { - error = avcodec_receive_frame(stream.codec, stream.frame.get()); - if (!error) { - //processReadFrame(); // #TODO streaming - return std::nullopt; - } - - if (error.code() != AVERROR(EAGAIN) || stream.queue.empty()) { - return error; - } - - const auto packet = &stream.queue.front().fields(); - const auto guard = gsl::finally([ - &, - size = packet->size, - data = packet->data - ] { - packet->size = size; - packet->data = data; - stream.queue.pop_front(); - }); - - error = avcodec_send_packet( - stream.codec, - packet->data ? packet : nullptr); // Drain on eof. - if (!error) { - continue; - } - LogError(qstr("avcodec_send_packet"), error); - if (error.code() == AVERROR_INVALIDDATA - // There is a sample voice message where skipping such packet - // results in a crash (read_access to nullptr) in swr_convert(). - && stream.codec->codec_id != AV_CODEC_ID_OPUS) { - if (++stream.invalidDataPackets < kSkipInvalidDataPackets) { - continue; // Try to skip a bad packet. - } - } - return error; - } while (true); - - [[unreachable]]; -} - -CodecPointer::CodecPointer(std::nullptr_t) { -} - -CodecPointer::CodecPointer(CodecPointer &&other) -: _context(base::take(other._context)) { -} - -CodecPointer &CodecPointer::operator=(CodecPointer &&other) { - if (this != &other) { - destroy(); - _context = base::take(other._context); - } - return *this; -} - -CodecPointer &CodecPointer::operator=(std::nullptr_t) { - destroy(); - return *this; -} - -void CodecPointer::destroy() { - if (_context) { - avcodec_free_context(&_context); - } -} - -CodecPointer CodecPointer::FromStream(not_null stream) { - auto error = AvErrorWrap(); - - auto result = CodecPointer(); - const auto context = result._context = avcodec_alloc_context3(nullptr); + auto result = CodecPointer(avcodec_alloc_context3(nullptr)); + const auto context = result.get(); if (!context) { LogError(qstr("avcodec_alloc_context3")); return {}; @@ -141,35 +90,230 @@ CodecPointer CodecPointer::FromStream(not_null stream) { return result; } -AVCodecContext *CodecPointer::get() const { - return _context; +void CodecDeleter::operator()(AVCodecContext *value) { + if (value) { + avcodec_free_context(&value); + } } -AVCodecContext *CodecPointer::operator->() const { - Expects(_context != nullptr); - - return get(); +FramePointer MakeFramePointer() { + return FramePointer(av_frame_alloc()); } -CodecPointer::operator AVCodecContext*() const { - return get(); -} - -AVCodecContext* CodecPointer::release() { - return base::take(_context); -} - -CodecPointer::~CodecPointer() { - destroy(); -} - -FrameDeleter::pointer FrameDeleter::create() { - return av_frame_alloc(); -} - -void FrameDeleter::operator()(pointer value) { +void FrameDeleter::operator()(AVFrame *value) { av_frame_free(&value); } +SwsContextPointer MakeSwsContextPointer( + not_null frame, + QSize resize, + SwsContextPointer *existing) { + const auto result = sws_getCachedContext( + existing ? existing->release() : nullptr, + frame->width, + frame->height, + AVPixelFormat(frame->format), + resize.width(), + resize.height(), + AV_PIX_FMT_BGRA, + 0, + nullptr, + nullptr, + nullptr); + if (!result) { + LogError(qstr("sws_getCachedContext")); + } + return SwsContextPointer(result); +} + +void SwsContextDeleter::operator()(SwsContext *value) { + if (value) { + sws_freeContext(value); + } +} + +void LogError(QLatin1String method) { + LOG(("Streaming Error: Error in %1.").arg(method)); +} + +void LogError(QLatin1String method, AvErrorWrap error) { + LOG(("Streaming Error: Error in %1 (code: %2, text: %3)." + ).arg(method + ).arg(error.code() + ).arg(error.text())); +} + +crl::time PtsToTime(int64_t pts, const AVRational &timeBase) { + return (pts == AV_NOPTS_VALUE) + ? Information::kDurationUnknown + : ((pts * 1000LL * timeBase.num) / timeBase.den); +} + +int ReadRotationFromMetadata(not_null stream) { + const auto tag = av_dict_get(stream->metadata, "rotate", nullptr, 0); + if (tag && *tag->value) { + const auto string = QString::fromUtf8(tag->value); + auto ok = false; + const auto degrees = string.toInt(&ok); + if (ok && (degrees == 90 || degrees == 180 || degrees == 270)) { + return degrees; + } + } + return 90; +} + +bool RotationSwapWidthHeight(int rotation) { + return (rotation == 90 || rotation == 270); +} + +std::optional ReadNextFrame(Stream &stream) { + Expects(stream.frame != nullptr); + + auto error = AvErrorWrap(); + + ClearFrameMemory(stream.frame.get()); + do { + error = avcodec_receive_frame(stream.codec.get(), stream.frame.get()); + if (!error) { + //processReadFrame(); // #TODO streaming + return std::nullopt; + } + + if (error.code() != AVERROR(EAGAIN) || stream.queue.empty()) { + return error; + } + + const auto packet = &stream.queue.front().fields(); + const auto guard = gsl::finally([ + &, + size = packet->size, + data = packet->data + ] { + packet->size = size; + packet->data = data; + stream.queue.pop_front(); + }); + + error = avcodec_send_packet( + stream.codec.get(), + packet->data ? packet : nullptr); // Drain on eof. + if (!error) { + continue; + } + LogError(qstr("avcodec_send_packet"), error); + if (error.code() == AVERROR_INVALIDDATA + // There is a sample voice message where skipping such packet + // results in a crash (read_access to nullptr) in swr_convert(). + && stream.codec->codec_id != AV_CODEC_ID_OPUS) { + if (++stream.invalidDataPackets < kSkipInvalidDataPackets) { + continue; // Try to skip a bad packet. + } + } + return error; + } while (true); + + [[unreachable]]; +} + +QImage ConvertFrame( + Stream &stream, + QSize resize, + QImage storage) { + Expects(stream.frame != nullptr); + + const auto frame = stream.frame.get(); + const auto frameSize = QSize(frame->width, frame->height); + if (frameSize.isEmpty()) { + LOG(("Streaming Error: Bad frame size %1,%2" + ).arg(resize.width() + ).arg(resize.height())); + return QImage(); + } else if (!frame->data[0]) { + LOG(("Streaming Error: Bad frame data.")); + return QImage(); + } + if (resize.isEmpty()) { + resize = frameSize; + } else if (RotationSwapWidthHeight(stream.rotation)) { + resize.transpose(); + } + if (storage.isNull() + || storage.size() != resize + || !storage.isDetached() + || !IsAlignedImage(storage)) { + storage = CreateAlignedImage(resize); + } + const auto format = AV_PIX_FMT_BGRA; + const auto hasDesiredFormat = (frame->format == format) + || ((frame->format == -1) && (stream.codec->pix_fmt == format)); + if (frameSize == storage.size() && hasDesiredFormat) { + static_assert(sizeof(uint32) == kPixelBytesSize); + auto to = reinterpret_cast(storage.bits()); + auto from = reinterpret_cast(frame->data[0]); + const auto deltaTo = (storage.bytesPerLine() / kPixelBytesSize) + - storage.width(); + const auto deltaFrom = (frame->linesize[0] / kPixelBytesSize) + - frame->width; + for (const auto y : ranges::view::ints(0, frame->height)) { + for (const auto x : ranges::view::ints(0, frame->width)) { + // Wipe out possible alpha values. + *to++ = 0x000000FFU | *from++; + } + to += deltaTo; + from += deltaFrom; + } + } else { + stream.swsContext = MakeSwsContextPointer( + frame, + resize, + &stream.swsContext); + if (!stream.swsContext) { + return QImage(); + } + + // AV_NUM_DATA_POINTERS defined in AVFrame struct + uint8_t *data[AV_NUM_DATA_POINTERS] = { storage.bits(), nullptr }; + int linesize[AV_NUM_DATA_POINTERS] = { storage.bytesPerLine(), 0 }; + + const auto lines = sws_scale( + stream.swsContext.get(), + frame->data, + frame->linesize, + 0, + frame->height, + data, + linesize); + if (lines != resize.height()) { + LOG(("Streaming Error: " + "Unable to sws_scale to good size %1, got %2." + ).arg(resize.height() + ).arg(lines)); + return QImage(); + } + } + if (stream.rotation == 180) { + storage = std::move(storage).mirrored(true, true); + } else if (stream.rotation != 0) { + auto transform = QTransform(); + transform.rotate(stream.rotation); + storage = storage.transformed(transform); + } + + // Read some future packets for audio stream. + //if (_audioStreamId >= 0) { + // while (_frameMs + 5000 > _lastReadAudioMs + // && _frameMs + 15000 > _lastReadVideoMs) { + // auto packetResult = readAndProcessPacket(); + // if (packetResult != PacketResult::Ok) { + // break; + // } + // } + //} + // #TODO streaming + + ClearFrameMemory(stream.frame.get()); + return std::move(storage); +} + } // namespace Streaming } // namespace Media diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_common.h b/Telegram/SourceFiles/media/streaming/media_streaming_common.h index d47fe7b07..6168b30a7 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_common.h +++ b/Telegram/SourceFiles/media/streaming/media_streaming_common.h @@ -10,13 +10,12 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL extern "C" { #include #include +#include } // extern "C" namespace Media { namespace Streaming { -[[nodiscard]] crl::time PtsToTime(int64_t pts, const AVRational &timeBase); - enum class Mode { Both, Audio, @@ -33,6 +32,7 @@ struct Information { crl::time started = 0; QImage cover; + int rotation = 0; }; class AvErrorWrap { @@ -61,9 +61,6 @@ private: }; -void LogError(QLatin1String method); -void LogError(QLatin1String method, AvErrorWrap error); - class Packet { public: Packet() { @@ -118,35 +115,26 @@ private: }; -class CodecPointer { -public: - CodecPointer(std::nullptr_t = nullptr); - CodecPointer(CodecPointer &&other); - CodecPointer &operator=(CodecPointer &&other); - CodecPointer &operator=(std::nullptr_t); - ~CodecPointer(); - - [[nodiscard]] static CodecPointer FromStream( - not_null stream); - - [[nodiscard]] AVCodecContext *get() const; - [[nodiscard]] AVCodecContext *operator->() const; - [[nodiscard]] operator AVCodecContext*() const; - [[nodiscard]] AVCodecContext* release(); - -private: - void destroy(); - - AVCodecContext *_context = nullptr; - +struct CodecDeleter { + void operator()(AVCodecContext *value); }; +using CodecPointer = std::unique_ptr; +CodecPointer MakeCodecPointer(not_null stream); struct FrameDeleter { - using pointer = AVFrame*; - [[nodiscard]] static pointer create(); - void operator()(pointer value); + void operator()(AVFrame *value); }; -using FramePointer = std::unique_ptr; +using FramePointer = std::unique_ptr; +FramePointer MakeFramePointer(); + +struct SwsContextDeleter { + void operator()(SwsContext *value); +}; +using SwsContextPointer = std::unique_ptr; +SwsContextPointer MakeSwsContextPointer( + not_null frame, + QSize resize, + SwsContextPointer *existing = nullptr); struct Stream { CodecPointer codec; @@ -154,9 +142,23 @@ struct Stream { std::deque queue; crl::time lastReadPositionTime = 0; int invalidDataPackets = 0; + + // Video only. + int rotation = 0; + SwsContextPointer swsContext; }; +void LogError(QLatin1String method); +void LogError(QLatin1String method, AvErrorWrap error); + +[[nodiscard]] crl::time PtsToTime(int64_t pts, const AVRational &timeBase); +[[nodiscard]] int ReadRotationFromMetadata(not_null stream); +[[nodiscard]] bool RotationSwapWidthHeight(int rotation); [[nodiscard]] std::optional ReadNextFrame(Stream &stream); +[[nodiscard]] QImage ConvertFrame( + Stream& stream, + QSize resize, + QImage storage); } // namespace Streaming } // namespace Media diff --git a/Telegram/SourceFiles/media/streaming/media_streaming_file.cpp b/Telegram/SourceFiles/media/streaming/media_streaming_file.cpp index e211ec86b..1b3492809 100644 --- a/Telegram/SourceFiles/media/streaming/media_streaming_file.cpp +++ b/Telegram/SourceFiles/media/streaming/media_streaming_file.cpp @@ -100,7 +100,7 @@ void File::Context::initStream(StreamWrap &wrap, AVMediaType type) { type, -1, -1, - 0, + nullptr, 0); if (wrap.id < 0) { return; @@ -108,27 +108,15 @@ void File::Context::initStream(StreamWrap &wrap, AVMediaType type) { wrap.info = _formatContext->streams[wrap.id]; if (type == AVMEDIA_TYPE_VIDEO) { - const auto rotateTag = av_dict_get( - wrap.info->metadata, - "rotate", - nullptr, - 0); - if (rotateTag && *rotateTag->value) { - const auto stringRotateTag = QString::fromUtf8(rotateTag->value); - auto toIntSucceeded = false; - const auto rotateDegrees = stringRotateTag.toInt(&toIntSucceeded); - if (toIntSucceeded) { - //_rotation = rotationFromDegrees(rotateDegrees); // #TODO streaming - } - } + wrap.stream.rotation = ReadRotationFromMetadata(wrap.info); } - wrap.stream.codec = CodecPointer::FromStream(wrap.info); + wrap.stream.codec = MakeCodecPointer(wrap.info); if (!wrap.stream.codec) { ClearStream(wrap); return; } - wrap.stream.frame.reset(FrameDeleter::create()); + wrap.stream.frame = MakeFramePointer(); if (!wrap.stream.frame) { ClearStream(wrap); return; @@ -289,6 +277,12 @@ void File::Context::readInformation(crl::time positionTime) { information.cover = readFirstVideoFrame(); if (unroll()) { return; + } else if (!information.cover.isNull()) { + information.video = information.cover.size(); + information.rotation = _video.stream.rotation; + if (RotationSwapWidthHeight(information.rotation)) { + information.video.transpose(); + } } information.audio = (_audio.info != nullptr); @@ -342,7 +336,7 @@ base::variant File::Context::tryReadFirstVideoFrame() { } return *error; } - return QImage(); // #TODO streaming decode frame + return ConvertFrame(_video.stream, QSize(), QImage()); } void File::Context::enqueueEofPackets() { @@ -373,7 +367,7 @@ void File::Context::processPacket(Packet &&packet) { return false; }; - check(_audio) && check(_video); + check(_audio) || check(_video); } void File::Context::readNextPacket() { @@ -445,9 +439,6 @@ File::Context::~Context() { ClearStream(_audio); ClearStream(_video); - //if (_swsContext) { - // sws_freeContext(_swsContext); - //} if (_opened) { avformat_close_input(&_formatContext); }