From e89350d4b71b0aedb3f5a1a64496ae4bc1c4f523 Mon Sep 17 00:00:00 2001 From: John Preston Date: Tue, 2 Jan 2018 20:22:13 +0300 Subject: [PATCH] Reuse resample code from FFMpegLoader for video. AbstractAudioFFMpegLoader used in FFMpegLoader and ChildFFMpegLoader. --- Telegram/SourceFiles/media/media_audio.cpp | 8 +- .../media/media_audio_ffmpeg_loader.cpp | 615 ++++++++++-------- .../media/media_audio_ffmpeg_loader.h | 69 +- .../media/media_child_ffmpeg_loader.cpp | 192 +----- .../media/media_child_ffmpeg_loader.h | 44 +- 5 files changed, 409 insertions(+), 519 deletions(-) diff --git a/Telegram/SourceFiles/media/media_audio.cpp b/Telegram/SourceFiles/media/media_audio.cpp index 38bb5d0fe..22c8d61fc 100644 --- a/Telegram/SourceFiles/media/media_audio.cpp +++ b/Telegram/SourceFiles/media/media_audio.cpp @@ -1542,7 +1542,9 @@ public: QByteArray buffer; buffer.reserve(AudioVoiceMsgBufferSize); - int64 countbytes = sampleSize * samplesCount(), processed = 0, sumbytes = 0; + int64 countbytes = sampleSize() * samplesCount(); + int64 processed = 0; + int64 sumbytes = 0; if (samplesCount() < Media::Player::kWaveformSamplesCount) { return false; } @@ -1552,7 +1554,7 @@ public: auto fmt = format(); auto peak = uint16(0); - auto callback = [&peak, &sumbytes, &peaks, countbytes](uint16 sample) { + auto callback = [&](uint16 sample) { accumulate_max(peak, sample); sumbytes += Media::Player::kWaveformSamplesCount; if (sumbytes >= countbytes) { @@ -1579,7 +1581,7 @@ public: } else if (fmt == AL_FORMAT_MONO16 || fmt == AL_FORMAT_STEREO16) { Media::Audio::IterateSamples(sampleBytes, callback); } - processed += sampleSize * samples; + processed += sampleSize() * samples; } if (sumbytes > 0 && peaks.size() < Media::Player::kWaveformSamplesCount) { peaks.push_back(peak); diff --git a/Telegram/SourceFiles/media/media_audio_ffmpeg_loader.cpp b/Telegram/SourceFiles/media/media_audio_ffmpeg_loader.cpp index 79b998d17..0e9f3143b 100644 --- a/Telegram/SourceFiles/media/media_audio_ffmpeg_loader.cpp +++ b/Telegram/SourceFiles/media/media_audio_ffmpeg_loader.cpp @@ -20,14 +20,6 @@ Copyright (c) 2014-2017 John Preston, https://desktop.telegram.org */ #include "media/media_audio_ffmpeg_loader.h" -namespace { - -constexpr AVSampleFormat AudioToFormat = AV_SAMPLE_FMT_S16; -constexpr int64_t AudioToChannelLayout = AV_CH_LAYOUT_STEREO; -constexpr int32 AudioToChannels = 2; - -} // namespace - uint64_t AbstractFFMpegLoader::ComputeChannelLayout( uint64_t channel_layout, int channels) { @@ -201,7 +193,7 @@ int64_t AbstractFFMpegLoader::_seek_file(void *opaque, int64_t offset, int whenc return -1; } -FFMpegLoader::FFMpegLoader( +AbstractAudioFFMpegLoader::AbstractAudioFFMpegLoader( const FileLocation &file, const QByteArray &data, base::byte_vector &&bytes) @@ -209,11 +201,326 @@ FFMpegLoader::FFMpegLoader( _frame = av_frame_alloc(); } +bool AbstractAudioFFMpegLoader::initUsingContext( + not_null context, + int64 initialCount, + int initialFrequency) { + const auto layout = ComputeChannelLayout( + context->channel_layout, + context->channels); + if (!layout) { + LOG(("Audio Error: Unknown channel layout %1 for %2 channels." + ).arg(context->channel_layout + ).arg(context->channels + )); + return false; + } + + _swrSrcSampleFormat = context->sample_fmt; + switch (layout) { + case AV_CH_LAYOUT_MONO: + switch (_swrSrcSampleFormat) { + case AV_SAMPLE_FMT_U8: + case AV_SAMPLE_FMT_U8P: + _swrDstSampleFormat = _swrSrcSampleFormat; + _swrDstChannelLayout = layout; + _outputChannels = 1; + _outputSampleSize = 1; + _outputFormat = AL_FORMAT_MONO8; + break; + case AV_SAMPLE_FMT_S16: + case AV_SAMPLE_FMT_S16P: + _swrDstSampleFormat = _swrSrcSampleFormat; + _swrDstChannelLayout = layout; + _outputChannels = 1; + _outputSampleSize = sizeof(uint16); + _outputFormat = AL_FORMAT_MONO16; + break; + } + break; + case AV_CH_LAYOUT_STEREO: + switch (_swrSrcSampleFormat) { + case AV_SAMPLE_FMT_U8: + _swrDstSampleFormat = _swrSrcSampleFormat; + _swrDstChannelLayout = layout; + _outputChannels = 2; + _outputSampleSize = 2; + _outputFormat = AL_FORMAT_STEREO8; + break; + case AV_SAMPLE_FMT_S16: + _swrDstSampleFormat = _swrSrcSampleFormat; + _swrDstChannelLayout = layout; + _outputChannels = 2; + _outputSampleSize = 2 * sizeof(uint16); + _outputFormat = AL_FORMAT_STEREO16; + break; + } + break; + } + + if (_swrDstRate == initialFrequency) { + _outputSamplesCount = initialCount; + } else { + _outputSamplesCount = av_rescale_rnd( + initialCount, + _swrDstRate, + initialFrequency, + AV_ROUND_UP); + } + return true; +} + +auto AbstractAudioFFMpegLoader::readFromReadyContext( + not_null context, + QByteArray &result, + int64 &samplesAdded) +-> ReadResult { + av_frame_unref(_frame); + const auto res = avcodec_receive_frame(context, _frame); + if (res >= 0) { + return readFromReadyFrame(result, samplesAdded); + } + + if (res == AVERROR_EOF) { + return ReadResult::EndOfFile; + } else if (res != AVERROR(EAGAIN)) { + char err[AV_ERROR_MAX_STRING_SIZE] = { 0 }; + LOG(("Audio Error: " + "Unable to avcodec_receive_frame() file '%1', data size '%2', " + "error %3, %4" + ).arg(_file.name() + ).arg(_data.size() + ).arg(res + ).arg(av_make_error_string(err, sizeof(err), res) + )); + return ReadResult::Error; + } + return ReadResult::Wait; +} + +bool AbstractAudioFFMpegLoader::frameHasDesiredFormat() const { + const auto frameChannelLayout = ComputeChannelLayout( + _frame->channel_layout, + _frame->channels); + return true + && (_frame->format == _swrDstSampleFormat) + && (frameChannelLayout == _swrDstChannelLayout) + && (_frame->sample_rate == _swrDstRate); +} + +bool AbstractAudioFFMpegLoader::initResampleForFrame() { + const auto frameChannelLayout = ComputeChannelLayout( + _frame->channel_layout, + _frame->channels); + if (!frameChannelLayout) { + LOG(("Audio Error: " + "Unable to compute channel layout for frame in file '%1', " + "data size '%2', channel_layout %3, channels %4" + ).arg(_file.name() + ).arg(_data.size() + ).arg(_frame->channel_layout + ).arg(_frame->channels + )); + return false; + } else if (_frame->format == -1) { + LOG(("Audio Error: " + "Unknown frame format in file '%1', data size '%2'" + ).arg(_file.name() + ).arg(_data.size() + )); + return false; + } else if (_swrContext) { + if (true + && (_frame->format == _swrSrcSampleFormat) + && (frameChannelLayout == _swrSrcChannelLayout) + && (_frame->sample_rate == _swrSrcRate)) { + return true; + } + swr_close(_swrContext); + } + + _swrSrcSampleFormat = static_cast(_frame->format); + _swrSrcChannelLayout = frameChannelLayout; + _swrSrcRate = _frame->sample_rate; + return initResampleUsingFormat(); +} + +bool AbstractAudioFFMpegLoader::initResampleUsingFormat() { + int res = 0; + + _swrContext = swr_alloc_set_opts( + _swrContext, + _swrDstChannelLayout, + _swrDstSampleFormat, + _swrDstRate, + _swrSrcChannelLayout, + _swrSrcSampleFormat, + _swrSrcRate, + 0, + nullptr); + if (!_swrContext) { + LOG(("Audio Error: " + "Unable to swr_alloc for file '%1', data size '%2'" + ).arg(_file.name() + ).arg(_data.size())); + return false; + } else if ((res = swr_init(_swrContext)) < 0) { + char err[AV_ERROR_MAX_STRING_SIZE] = { 0 }; + LOG(("Audio Error: " + "Unable to swr_init for file '%1', data size '%2', " + "error %3, %4" + ).arg(_file.name() + ).arg(_data.size() + ).arg(res + ).arg(av_make_error_string(err, sizeof(err), res) + )); + return false; + } + if (_swrDstData) { + av_freep(&_swrDstData[0]); + _swrDstDataCapacity = -1; + } + return true; +} + +bool AbstractAudioFFMpegLoader::ensureResampleSpaceAvailable(int samples) { + if (_swrDstData != nullptr && _swrDstDataCapacity >= samples) { + return true; + } + const auto allocate = std::max(samples, int(av_rescale_rnd( + AVBlockSize / _outputSampleSize, + _swrDstRate, + _swrSrcRate, + AV_ROUND_UP))); + + if (_swrDstData) { + av_freep(&_swrDstData[0]); + } + const auto res = _swrDstData + ? av_samples_alloc( + _swrDstData, + nullptr, + _outputChannels, + allocate, + _swrDstSampleFormat, + 0) + : av_samples_alloc_array_and_samples( + &_swrDstData, + nullptr, + _outputChannels, + allocate, + _swrDstSampleFormat, + 0); + if (res < 0) { + char err[AV_ERROR_MAX_STRING_SIZE] = { 0 }; + LOG(("Audio Error: " + "Unable to av_samples_alloc for file '%1', data size '%2', " + "error %3, %4" + ).arg(_file.name() + ).arg(_data.size() + ).arg(res + ).arg(av_make_error_string(err, sizeof(err), res) + )); + return false; + } + _swrDstDataCapacity = allocate; + return true; +} + +void AbstractAudioFFMpegLoader::appendSamples( + QByteArray &result, + int64 &samplesAdded, + uint8_t **data, + int count) const { + result.append( + reinterpret_cast(data[0]), + count * _outputSampleSize); + samplesAdded += count; +} + +AudioPlayerLoader::ReadResult AbstractAudioFFMpegLoader::readFromReadyFrame( + QByteArray &result, + int64 &samplesAdded) { + if (frameHasDesiredFormat()) { + appendSamples( + result, + samplesAdded, + _frame->extended_data, + _frame->nb_samples); + return ReadResult::Ok; + } else if (!initResampleForFrame()) { + return ReadResult::Error; + } + + const auto maxSamples = av_rescale_rnd( + swr_get_delay(_swrContext, _swrSrcRate) + _frame->nb_samples, + _swrDstRate, + _swrSrcRate, + AV_ROUND_UP); + if (!ensureResampleSpaceAvailable(maxSamples)) { + return ReadResult::Error; + } + const auto samples = swr_convert( + _swrContext, + _swrDstData, + maxSamples, + (const uint8_t**)_frame->extended_data, + _frame->nb_samples); + if (samples < 0) { + char err[AV_ERROR_MAX_STRING_SIZE] = { 0 }; + LOG(("Audio Error: " + "Unable to swr_convert for file '%1', data size '%2', " + "error %3, %4" + ).arg(_file.name() + ).arg(_data.size() + ).arg(samples + ).arg(av_make_error_string(err, sizeof(err), samples) + )); + return ReadResult::Error; + } + + appendSamples( + result, + samplesAdded, + _swrDstData, + samples); + return ReadResult::Ok; +} + +AbstractAudioFFMpegLoader::~AbstractAudioFFMpegLoader() { + if (_swrContext) { + swr_free(&_swrContext); + } + if (_swrDstData) { + if (_swrDstData[0]) { + av_freep(&_swrDstData[0]); + } + av_freep(&_swrDstData); + } + av_frame_free(&_frame); +} + +FFMpegLoader::FFMpegLoader( + const FileLocation &file, + const QByteArray &data, + base::byte_vector &&bytes) +: AbstractAudioFFMpegLoader(file, data, std::move(bytes)) { +} + bool FFMpegLoader::open(TimeMs positionMs) { if (!AbstractFFMpegLoader::open(positionMs)) { return false; } + if (!openCodecContext()) { + return false; + } + if (!initUsingContext(_codecContext, _samplesCount, _samplesFrequency)) { + return false; + } + return seekTo(positionMs); +} +bool FFMpegLoader::openCodecContext() { int res = 0; char err[AV_ERROR_MAX_STRING_SIZE] = { 0 }; @@ -229,8 +536,8 @@ bool FFMpegLoader::open(TimeMs positionMs) { const auto stream = fmtContext->streams[streamId]; if ((res = avcodec_parameters_to_context( - _codecContext, - stream->codecpar)) < 0) { + _codecContext, + stream->codecpar)) < 0) { LOG(("Audio Error: " "Unable to avcodec_parameters_to_context for file '%1', " "data size '%2', error %3, %4" @@ -255,71 +562,12 @@ bool FFMpegLoader::open(TimeMs positionMs) { )); return false; } + return true; +} - const auto layout = ComputeChannelLayout( - _codecContext->channel_layout, - _codecContext->channels); - if (!layout) { - LOG(("Audio Error: Unknown channel layout %1 for %2 channels." - ).arg(_codecContext->channel_layout - ).arg(_codecContext->channels - )); - return false; - } - - _swrSrcFormat = _codecContext->sample_fmt; - switch (layout) { - case AV_CH_LAYOUT_MONO: - switch (_swrSrcFormat) { - case AV_SAMPLE_FMT_U8: - case AV_SAMPLE_FMT_U8P: - _swrDstFormat = _swrSrcFormat; - _swrDstChannelLayout = layout; - _swrDstChannels = 1; - _format = AL_FORMAT_MONO8; - sampleSize = 1; - break; - case AV_SAMPLE_FMT_S16: - case AV_SAMPLE_FMT_S16P: - _swrDstFormat = _swrSrcFormat; - _swrDstChannelLayout = layout; - _swrDstChannels = 1; - _format = AL_FORMAT_MONO16; - sampleSize = sizeof(uint16); - break; - } - break; - case AV_CH_LAYOUT_STEREO: - switch (_swrSrcFormat) { - case AV_SAMPLE_FMT_U8: - _swrDstFormat = _swrSrcFormat; - _swrDstChannelLayout = layout; - _swrDstChannels = 2; - _format = AL_FORMAT_STEREO8; - sampleSize = 2; - break; - case AV_SAMPLE_FMT_S16: - _swrDstFormat = _swrSrcFormat; - _swrDstChannelLayout = layout; - _swrDstChannels = 2; - _format = AL_FORMAT_STEREO16; - sampleSize = 2 * sizeof(uint16); - break; - } - break; - } - - if (_swrDstRate == _samplesFrequency) { - _swrDstSamplesCount = _samplesCount; - } else { - _swrDstSamplesCount = av_rescale_rnd( - _samplesCount, - _swrDstRate, - _samplesFrequency, - AV_ROUND_UP); - } - +bool FFMpegLoader::seekTo(TimeMs positionMs) { if (positionMs) { + const auto stream = fmtContext->streams[streamId]; const auto timeBase = stream->time_base; const auto timeStamp = (positionMs * timeBase.den) / (1000LL * timeBase.num); @@ -337,29 +585,15 @@ bool FFMpegLoader::open(TimeMs positionMs) { AudioPlayerLoader::ReadResult FFMpegLoader::readMore( QByteArray &result, int64 &samplesAdded) { - int res; - - av_frame_unref(_frame); - res = avcodec_receive_frame(_codecContext, _frame); - if (res >= 0) { - return readFromReadyFrame(result, samplesAdded); - } - - if (res == AVERROR_EOF) { - return ReadResult::EndOfFile; - } else if (res != AVERROR(EAGAIN)) { - char err[AV_ERROR_MAX_STRING_SIZE] = { 0 }; - LOG(("Audio Error: " - "Unable to avcodec_receive_frame() file '%1', data size '%2', " - "error %3, %4" - ).arg(_file.name() - ).arg(_data.size() - ).arg(res - ).arg(av_make_error_string(err, sizeof(err), res) - )); - return ReadResult::Error; + const auto readResult = readFromReadyContext( + _codecContext, + result, + samplesAdded); + if (readResult != ReadResult::Wait) { + return readResult; } + auto res = 0; if ((res = av_read_frame(fmtContext, &_packet)) < 0) { if (res != AVERROR_EOF) { char err[AV_ERROR_MAX_STRING_SIZE] = { 0 }; @@ -403,199 +637,8 @@ AudioPlayerLoader::ReadResult FFMpegLoader::readMore( return ReadResult::Ok; } -bool FFMpegLoader::frameHasDesiredFormat() const { - const auto frameChannelLayout = ComputeChannelLayout( - _frame->channel_layout, - _frame->channels); - return true - && (_frame->format == _swrDstFormat) - && (frameChannelLayout == _swrDstChannelLayout) - && (_frame->sample_rate == _swrDstRate); -} - -bool FFMpegLoader::initResampleForFrame() { - const auto frameChannelLayout = ComputeChannelLayout( - _frame->channel_layout, - _frame->channels); - if (!frameChannelLayout) { - LOG(("Audio Error: " - "Unable to compute channel layout for frame in file '%1', " - "data size '%2', channel_layout %3, channels %4" - ).arg(_file.name() - ).arg(_data.size() - ).arg(_frame->channel_layout - ).arg(_frame->channels - )); - return false; - } else if (_frame->format == -1) { - LOG(("Audio Error: " - "Unknown frame format in file '%1', data size '%2'" - ).arg(_file.name() - ).arg(_data.size() - )); - return false; - } else if (_swrContext) { - if (true - && (_frame->format == _swrSrcFormat) - && (frameChannelLayout == _swrSrcChannelLayout) - && (_frame->sample_rate == _swrSrcRate)) { - return true; - } - swr_close(_swrContext); - } - - _swrSrcFormat = static_cast(_frame->format); - _swrSrcChannelLayout = frameChannelLayout; - _swrSrcRate = _frame->sample_rate; - return initResampleUsingFormat(); -} - -bool FFMpegLoader::initResampleUsingFormat() { - int res = 0; - - _swrContext = swr_alloc_set_opts( - _swrContext, - _swrDstChannelLayout, - _swrDstFormat, - _swrDstRate, - _swrSrcChannelLayout, - _swrSrcFormat, - _swrSrcRate, - 0, - nullptr); - if (!_swrContext) { - LOG(("Audio Error: " - "Unable to swr_alloc for file '%1', data size '%2'" - ).arg(_file.name() - ).arg(_data.size())); - return false; - } else if ((res = swr_init(_swrContext)) < 0) { - char err[AV_ERROR_MAX_STRING_SIZE] = { 0 }; - LOG(("Audio Error: " - "Unable to swr_init for file '%1', data size '%2', " - "error %3, %4" - ).arg(_file.name() - ).arg(_data.size() - ).arg(res - ).arg(av_make_error_string(err, sizeof(err), res) - )); - return false; - } - if (_swrDstData) { - av_freep(&_swrDstData[0]); - _swrDstDataCapacity = -1; - } - return true; -} - -bool FFMpegLoader::ensureResampleSpaceAvailable(int samples) { - if (_swrDstData != nullptr && _swrDstDataCapacity >= samples) { - return true; - } - const auto allocate = std::max(samples, int(av_rescale_rnd( - AVBlockSize / sampleSize, - _swrDstRate, - _swrSrcRate, - AV_ROUND_UP))); - - if (_swrDstData) { - av_freep(&_swrDstData[0]); - } - const auto res = _swrDstData - ? av_samples_alloc( - _swrDstData, - nullptr, - _swrDstChannels, - allocate, - _swrDstFormat, - 0) - : av_samples_alloc_array_and_samples( - &_swrDstData, - nullptr, - _swrDstChannels, - allocate, - _swrDstFormat, - 0); - if (res < 0) { - char err[AV_ERROR_MAX_STRING_SIZE] = { 0 }; - LOG(("Audio Error: " - "Unable to av_samples_alloc for file '%1', data size '%2', " - "error %3, %4" - ).arg(_file.name() - ).arg(_data.size() - ).arg(res - ).arg(av_make_error_string(err, sizeof(err), res) - )); - return false; - } - _swrDstDataCapacity = allocate; - return true; -} - -AudioPlayerLoader::ReadResult FFMpegLoader::readFromReadyFrame( - QByteArray &result, - int64 &samplesAdded) { - if (frameHasDesiredFormat()) { - result.append( - reinterpret_cast(_frame->extended_data[0]), - _frame->nb_samples * sampleSize); - samplesAdded += _frame->nb_samples; - } else if (!initResampleForFrame()) { - return ReadResult::Error; - } - - const auto maxSamples = av_rescale_rnd( - swr_get_delay(_swrContext, _swrSrcRate) + _frame->nb_samples, - _swrDstRate, - _swrSrcRate, - AV_ROUND_UP); - if (!ensureResampleSpaceAvailable(maxSamples)) { - return ReadResult::Error; - } - const auto samples = swr_convert( - _swrContext, - _swrDstData, - maxSamples, - (const uint8_t**)_frame->extended_data, - _frame->nb_samples); - if (samples < 0) { - char err[AV_ERROR_MAX_STRING_SIZE] = { 0 }; - LOG(("Audio Error: " - "Unable to swr_convert for file '%1', data size '%2', " - "error %3, %4" - ).arg(_file.name() - ).arg(_data.size() - ).arg(samples - ).arg(av_make_error_string(err, sizeof(err), samples) - )); - return ReadResult::Error; - } - - const auto bytesCount = av_samples_get_buffer_size( - nullptr, - _swrDstChannels, - samples, - _swrDstFormat, - 1); - result.append( - reinterpret_cast(_swrDstData[0]), - bytesCount); - samplesAdded += bytesCount / sampleSize; - return ReadResult::Ok; -} - FFMpegLoader::~FFMpegLoader() { if (_codecContext) { avcodec_free_context(&_codecContext); } - if (_swrContext) { - swr_free(&_swrContext); - } - if (_swrDstData) { - if (_swrDstData[0]) { - av_freep(&_swrDstData[0]); - } - av_freep(&_swrDstData); - } - av_frame_free(&_frame); } diff --git a/Telegram/SourceFiles/media/media_audio_ffmpeg_loader.h b/Telegram/SourceFiles/media/media_audio_ffmpeg_loader.h index cef1a6175..4a3cc7b89 100644 --- a/Telegram/SourceFiles/media/media_audio_ffmpeg_loader.h +++ b/Telegram/SourceFiles/media/media_audio_ffmpeg_loader.h @@ -81,17 +81,15 @@ private: }; -class FFMpegLoader : public AbstractFFMpegLoader { +class AbstractAudioFFMpegLoader : public AbstractFFMpegLoader { public: - FFMpegLoader( + AbstractAudioFFMpegLoader( const FileLocation &file, const QByteArray &data, base::byte_vector &&bytes); - bool open(TimeMs positionMs) override; - int64 samplesCount() override { - return _swrDstSamplesCount; + return _outputSamplesCount; } int samplesFrequency() override { @@ -99,15 +97,24 @@ public: } int format() override { - return _format; + return _outputFormat; } - ReadResult readMore(QByteArray &result, int64 &samplesAdded) override; - - ~FFMpegLoader(); + ~AbstractAudioFFMpegLoader(); protected: - int sampleSize = 2 * sizeof(uint16); + bool initUsingContext( + not_null context, + int64 initialCount, + int initialFrequency); + ReadResult readFromReadyContext( + not_null context, + QByteArray &result, + int64 &samplesAdded); + + int sampleSize() const { + return _outputSampleSize; + } private: ReadResult readFromReadyFrame(QByteArray &result, int64 &samplesAdded); @@ -116,24 +123,50 @@ private: bool initResampleUsingFormat(); bool ensureResampleSpaceAvailable(int samples); - AVCodecContext *_codecContext = nullptr; - AVPacket _packet; - int _format = AL_FORMAT_STEREO16; + void appendSamples( + QByteArray &result, + int64 &samplesAdded, + uint8_t **data, + int count) const; + AVFrame *_frame = nullptr; + int _outputFormat = AL_FORMAT_STEREO16; + int _outputChannels = 2; + int _outputSampleSize = 2 * sizeof(uint16); + int64 _outputSamplesCount = 0; SwrContext *_swrContext = nullptr; int _swrSrcRate = 0; - AVSampleFormat _swrSrcFormat = AV_SAMPLE_FMT_NONE; + AVSampleFormat _swrSrcSampleFormat = AV_SAMPLE_FMT_NONE; uint64_t _swrSrcChannelLayout = 0; const int _swrDstRate = Media::Player::kDefaultFrequency; - AVSampleFormat _swrDstFormat = AV_SAMPLE_FMT_S16; + AVSampleFormat _swrDstSampleFormat = AV_SAMPLE_FMT_S16; uint64_t _swrDstChannelLayout = AV_CH_LAYOUT_STEREO; - int _swrDstChannels = 2; - - int64 _swrDstSamplesCount = 0; uint8_t **_swrDstData = nullptr; int _swrDstDataCapacity = 0; }; + +class FFMpegLoader : public AbstractAudioFFMpegLoader { +public: + FFMpegLoader( + const FileLocation &file, + const QByteArray &data, + base::byte_vector &&bytes); + + bool open(TimeMs positionMs) override; + + ReadResult readMore(QByteArray &result, int64 &samplesAdded) override; + + ~FFMpegLoader(); + +private: + bool openCodecContext(); + bool seekTo(TimeMs positionMs); + + AVCodecContext *_codecContext = nullptr; + AVPacket _packet; + +}; diff --git a/Telegram/SourceFiles/media/media_child_ffmpeg_loader.cpp b/Telegram/SourceFiles/media/media_child_ffmpeg_loader.cpp index cef1911cf..e42ab0bbe 100644 --- a/Telegram/SourceFiles/media/media_child_ffmpeg_loader.cpp +++ b/Telegram/SourceFiles/media/media_child_ffmpeg_loader.cpp @@ -47,107 +47,30 @@ VideoSoundData::~VideoSoundData() { } } -ChildFFMpegLoader::ChildFFMpegLoader(std::unique_ptr &&data) : AudioPlayerLoader(FileLocation(), QByteArray(), base::byte_vector()) +ChildFFMpegLoader::ChildFFMpegLoader(std::unique_ptr &&data) +: AbstractAudioFFMpegLoader( + FileLocation(), + QByteArray(), + base::byte_vector()) , _parentData(std::move(data)) { - _frame = av_frame_alloc(); } bool ChildFFMpegLoader::open(TimeMs positionMs) { - int res = 0; - char err[AV_ERROR_MAX_STRING_SIZE] = { 0 }; - - auto layout = _parentData->context->channel_layout; - if (!layout) { - auto channelsCount = _parentData->context->channels; - switch (channelsCount) { - case 1: layout = AV_CH_LAYOUT_MONO; break; - case 2: layout = AV_CH_LAYOUT_STEREO; break; - default: LOG(("Audio Error: Unknown channel layout for %1 channels.").arg(channelsCount)); break; - } - } - _inputFormat = _parentData->context->sample_fmt; - switch (layout) { - case AV_CH_LAYOUT_MONO: - switch (_inputFormat) { - case AV_SAMPLE_FMT_U8: - case AV_SAMPLE_FMT_U8P: _format = AL_FORMAT_MONO8; _sampleSize = 1; break; - case AV_SAMPLE_FMT_S16: - case AV_SAMPLE_FMT_S16P: _format = AL_FORMAT_MONO16; _sampleSize = sizeof(uint16); break; - default: - _sampleSize = -1; // convert needed - break; - } - break; - case AV_CH_LAYOUT_STEREO: - switch (_inputFormat) { - case AV_SAMPLE_FMT_U8: _format = AL_FORMAT_STEREO8; _sampleSize = 2; break; - case AV_SAMPLE_FMT_S16: _format = AL_FORMAT_STEREO16; _sampleSize = 2 * sizeof(uint16); break; - default: - _sampleSize = -1; // convert needed - break; - } - break; - default: - _sampleSize = -1; // convert needed - break; - } - if (_parentData->frequency != Media::Player::kDefaultFrequency) { - _sampleSize = -1; // convert needed - } - - if (_sampleSize < 0) { - _swrContext = swr_alloc(); - if (!_swrContext) { - LOG(("Audio Error: Unable to swr_alloc for file '%1', data size '%2'").arg(_file.name()).arg(_data.size())); - return false; - } - int64_t src_ch_layout = layout, dst_ch_layout = AudioToChannelLayout; - _srcRate = _parentData->frequency; - AVSampleFormat src_sample_fmt = _inputFormat, dst_sample_fmt = AudioToFormat; - _dstRate = Media::Player::kDefaultFrequency; - - av_opt_set_int(_swrContext, "in_channel_layout", src_ch_layout, 0); - av_opt_set_int(_swrContext, "in_sample_rate", _srcRate, 0); - av_opt_set_sample_fmt(_swrContext, "in_sample_fmt", src_sample_fmt, 0); - av_opt_set_int(_swrContext, "out_channel_layout", dst_ch_layout, 0); - av_opt_set_int(_swrContext, "out_sample_rate", _dstRate, 0); - av_opt_set_sample_fmt(_swrContext, "out_sample_fmt", dst_sample_fmt, 0); - - if ((res = swr_init(_swrContext)) < 0) { - LOG(("Audio Error: Unable to swr_init for file '%1', data size '%2', error %3, %4").arg(_file.name()).arg(_data.size()).arg(res).arg(av_make_error_string(err, sizeof(err), res))); - return false; - } - - _sampleSize = AudioToChannels * sizeof(short); - _parentData->frequency = _dstRate; - _parentData->length = av_rescale_rnd(_parentData->length, _dstRate, _srcRate, AV_ROUND_UP); - _format = AL_FORMAT_STEREO16; - - _maxResampleSamples = av_rescale_rnd(AVBlockSize / _sampleSize, _dstRate, _srcRate, AV_ROUND_UP); - if ((res = av_samples_alloc_array_and_samples(&_dstSamplesData, 0, AudioToChannels, _maxResampleSamples, AudioToFormat, 0)) < 0) { - LOG(("Audio Error: Unable to av_samples_alloc for file '%1', data size '%2', error %3, %4").arg(_file.name()).arg(_data.size()).arg(res).arg(av_make_error_string(err, sizeof(err), res))); - return false; - } - } - - return true; + return initUsingContext( + _parentData->context, + _parentData->length, + _parentData->frequency); } -AudioPlayerLoader::ReadResult ChildFFMpegLoader::readMore(QByteArray &result, int64 &samplesAdded) { - int res; - - av_frame_unref(_frame); - res = avcodec_receive_frame(_parentData->context, _frame); - if (res >= 0) { - return readFromReadyFrame(result, samplesAdded); - } - - if (res == AVERROR_EOF) { - return ReadResult::EndOfFile; - } else if (res != AVERROR(EAGAIN)) { - char err[AV_ERROR_MAX_STRING_SIZE] = { 0 }; - LOG(("Audio Error: Unable to avcodec_receive_frame() file '%1', data size '%2', error %3, %4").arg(_file.name()).arg(_data.size()).arg(res).arg(av_make_error_string(err, sizeof(err), res))); - return ReadResult::Error; +AudioPlayerLoader::ReadResult ChildFFMpegLoader::readMore( + QByteArray &result, + int64 &samplesAdded) { + const auto readResult = readFromReadyContext( + _parentData->context, + result, + samplesAdded); + if (readResult != ReadResult::Wait) { + return readResult; } if (_queue.isEmpty()) { @@ -163,7 +86,7 @@ AudioPlayerLoader::ReadResult ChildFFMpegLoader::readMore(QByteArray &result, in return ReadResult::Ok; } - res = avcodec_send_packet(_parentData->context, &packet); + auto res = avcodec_send_packet(_parentData->context, &packet); if (res < 0) { FFMpeg::freePacket(&packet); @@ -180,90 +103,15 @@ AudioPlayerLoader::ReadResult ChildFFMpegLoader::readMore(QByteArray &result, in return ReadResult::Ok; } -AudioPlayerLoader::ReadResult ChildFFMpegLoader::readFromReadyFrame(QByteArray &result, int64 &samplesAdded) { - int res = 0; - - if (_dstSamplesData) { // convert needed - int64_t dstSamples = av_rescale_rnd(swr_get_delay(_swrContext, _srcRate) + _frame->nb_samples, _dstRate, _srcRate, AV_ROUND_UP); - if (dstSamples > _maxResampleSamples) { - _maxResampleSamples = dstSamples; - av_freep(&_dstSamplesData[0]); - if ((res = av_samples_alloc(_dstSamplesData, 0, AudioToChannels, _maxResampleSamples, AudioToFormat, 1)) < 0) { - char err[AV_ERROR_MAX_STRING_SIZE] = { 0 }; - LOG(("Audio Error: Unable to av_samples_alloc for file '%1', data size '%2', error %3, %4").arg(_file.name()).arg(_data.size()).arg(res).arg(av_make_error_string(err, sizeof(err), res))); - return ReadResult::Error; - } - } - - // See the same check in media_audio_ffmpeg_loader.cpp. - if (_frame->extended_data[1] == nullptr) { - const auto params = _parentData->context; - if (IsPlanarFormat(params->sample_fmt) && params->channels > 1) { - LOG(("Audio Error: Inconsistent frame layout/channels in file, codec: (%1;%2;%3), frame: (%4;%5;%6)." - ).arg(params->channel_layout - ).arg(params->channels - ).arg(params->sample_fmt - ).arg(_frame->channel_layout - ).arg(_frame->channels - ).arg(_frame->format - )); - return ReadResult::Error; - } else { - const auto key = "ffmpeg_" + std::to_string(ptrdiff_t(this)); - const auto value = QString("codec: (%1;%2;%3), frame: (%4;%5;%6), ptrs: (%7;%8;%9)" - ).arg(params->channel_layout - ).arg(params->channels - ).arg(params->sample_fmt - ).arg(_frame->channel_layout - ).arg(_frame->channels - ).arg(_frame->format - ).arg(ptrdiff_t(_frame->data[0]) - ).arg(ptrdiff_t(_frame->extended_data[0]) - ).arg(ptrdiff_t(_frame->data[1]) - ); - CrashReports::SetAnnotation(key, value); - } - } - - if ((res = swr_convert(_swrContext, _dstSamplesData, dstSamples, (const uint8_t**)_frame->extended_data, _frame->nb_samples)) < 0) { - char err[AV_ERROR_MAX_STRING_SIZE] = { 0 }; - LOG(("Audio Error: Unable to swr_convert for file '%1', data size '%2', error %3, %4").arg(_file.name()).arg(_data.size()).arg(res).arg(av_make_error_string(err, sizeof(err), res))); - return ReadResult::Error; - } - - if (_frame->extended_data[1] == nullptr) { - const auto key = "ffmpeg_" + std::to_string(ptrdiff_t(this)); - CrashReports::ClearAnnotation(key); - } - - int32 resultLen = av_samples_get_buffer_size(0, AudioToChannels, res, AudioToFormat, 1); - result.append((const char*)_dstSamplesData[0], resultLen); - samplesAdded += resultLen / _sampleSize; - } else { - result.append((const char*)_frame->extended_data[0], _frame->nb_samples * _sampleSize); - samplesAdded += _frame->nb_samples; - } - return ReadResult::Ok; -} - void ChildFFMpegLoader::enqueuePackets(QQueue &packets) { _queue += std::move(packets); packets.clear(); } ChildFFMpegLoader::~ChildFFMpegLoader() { - auto queue = base::take(_queue); - for (auto &packetData : queue) { + for (auto &packetData : base::take(_queue)) { AVPacket packet; FFMpeg::packetFromDataWrap(packet, packetData); FFMpeg::freePacket(&packet); } - if (_swrContext) swr_free(&_swrContext); - if (_dstSamplesData) { - if (_dstSamplesData[0]) { - av_freep(&_dstSamplesData[0]); - } - av_freep(&_dstSamplesData); - } - av_frame_free(&_frame); } diff --git a/Telegram/SourceFiles/media/media_child_ffmpeg_loader.h b/Telegram/SourceFiles/media/media_child_ffmpeg_loader.h index a3ca7fb8d..5f027e58f 100644 --- a/Telegram/SourceFiles/media/media_child_ffmpeg_loader.h +++ b/Telegram/SourceFiles/media/media_child_ffmpeg_loader.h @@ -20,17 +20,7 @@ Copyright (c) 2014-2017 John Preston, https://desktop.telegram.org */ #pragma once -#include "media/media_audio_loader.h" -#include "media/media_audio.h" - -extern "C" { -#include -#include -#include -#include -} // extern "C" - -#include +#include "media/media_audio_ffmpeg_loader.h" struct VideoSoundData { AVCodecContext *context = nullptr; @@ -81,7 +71,7 @@ inline void freePacket(AVPacket *packet) { } // namespace FFMpeg -class ChildFFMpegLoader : public AudioPlayerLoader { +class ChildFFMpegLoader : public AbstractAudioFFMpegLoader { public: ChildFFMpegLoader(std::unique_ptr &&data); @@ -91,18 +81,6 @@ public: return true; } - int format() override { - return _format; - } - - int64 samplesCount() override { - return _parentData->length; - } - - int32 samplesFrequency() override { - return _parentData->frequency; - } - ReadResult readMore(QByteArray &result, int64 &samplesAdded) override; void enqueuePackets(QQueue &packets) override; @@ -113,22 +91,8 @@ public: ~ChildFFMpegLoader(); private: - ReadResult readFromReadyFrame(QByteArray &result, int64 &samplesAdded); - + std::unique_ptr _parentData; + QQueue _queue; bool _eofReached = false; - int32 _sampleSize = 2 * sizeof(uint16); - int _format = AL_FORMAT_STEREO16; - int32 _srcRate = Media::Player::kDefaultFrequency; - int32 _dstRate = Media::Player::kDefaultFrequency; - int32 _maxResampleSamples = 1024; - uint8_t **_dstSamplesData = nullptr; - - std::unique_ptr _parentData; - AVSampleFormat _inputFormat; - AVFrame *_frame = nullptr; - - SwrContext *_swrContext = nullptr; - QQueue _queue; - };