procxx
/
kepka
mirror of https://github.com/procxx/kepka.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Correctly track DC index in Downloader-s.

This commit is contained in:
John Preston 2019-12-03 15:45:35 +03:00
parent 8535a579ca
commit fb86bb579b
6 changed files with 123 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Telegram/SourceFiles/main/main_session.cpp
View File

@ -44,7 +44,7 @@ Session::Session(
, _autoLockTimer([=] { checkAutoLock(); }) , _autoLockTimer([=] { checkAutoLock(); })
, _api(std::make_unique<ApiWrap>(this)) , _api(std::make_unique<ApiWrap>(this))
, _calls(std::make_unique<Calls::Instance>(this)) , _calls(std::make_unique<Calls::Instance>(this))
, _downloader(std::make_unique<Storage::Downloader>(_api.get())) , _downloader(std::make_unique<Storage::DownloadManager>(_api.get()))
, _uploader(std::make_unique<Storage::Uploader>(_api.get())) , _uploader(std::make_unique<Storage::Uploader>(_api.get()))
, _storage(std::make_unique<Storage::Facade>()) , _storage(std::make_unique<Storage::Facade>())
, _notifications(std::make_unique<Window::Notifications::System>(this)) , _notifications(std::make_unique<Window::Notifications::System>(this))

6
Telegram/SourceFiles/main/main_session.h
View File

@ -29,7 +29,7 @@ class Session;
} // namespace Data } // namespace Data
namespace Storage { namespace Storage {
class Downloader; class DownloadManager;
class Uploader; class Uploader;
class Facade; class Facade;
} // namespace Storage } // namespace Storage
@ -80,7 +80,7 @@ public:
} }
bool validateSelf(const MTPUser &user); bool validateSelf(const MTPUser &user);
[[nodiscard]] Storage::Downloader &downloader() { [[nodiscard]] Storage::DownloadManager &downloader() {
return *_downloader; return *_downloader;
} }
[[nodiscard]] Storage::Uploader &uploader() { [[nodiscard]] Storage::Uploader &uploader() {
@ -145,7 +145,7 @@ private:
const std::unique_ptr<ApiWrap> _api; const std::unique_ptr<ApiWrap> _api;
const std::unique_ptr<Calls::Instance> _calls; const std::unique_ptr<Calls::Instance> _calls;
const std::unique_ptr<Storage::Downloader> _downloader; const std::unique_ptr<Storage::DownloadManager> _downloader;
const std::unique_ptr<Storage::Uploader> _uploader; const std::unique_ptr<Storage::Uploader> _uploader;
const std::unique_ptr<Storage::Facade> _storage; const std::unique_ptr<Storage::Facade> _storage;
const std::unique_ptr<Window::Notifications::System> _notifications; const std::unique_ptr<Window::Notifications::System> _notifications;

2
Telegram/SourceFiles/media/streaming/media_streaming_loader_mtproto.cpp
View File

@ -21,7 +21,7 @@ constexpr auto kMaxConcurrentRequests = 4;
} // namespace } // namespace
LoaderMtproto::LoaderMtproto( LoaderMtproto::LoaderMtproto(
not_null<Storage::Downloader*> owner, not_null<Storage::DownloadManager*> owner,
const StorageFileLocation &location, const StorageFileLocation &location,
int size, int size,
Data::FileOrigin origin) Data::FileOrigin origin)

6
Telegram/SourceFiles/media/streaming/media_streaming_loader_mtproto.h
View File

@ -12,7 +12,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
#include "data/data_file_origin.h" #include "data/data_file_origin.h"
namespace Storage { namespace Storage {
class Downloader; class DownloadManager;
} // namespace Storage } // namespace Storage
namespace Media { namespace Media {
@ -21,7 +21,7 @@ namespace Streaming {
class LoaderMtproto : public Loader, public base::has_weak_ptr { class LoaderMtproto : public Loader, public base::has_weak_ptr {
public: public:
LoaderMtproto( LoaderMtproto(
not_null<Storage::Downloader*> owner, not_null<Storage::DownloadManager*> owner,
const StorageFileLocation &location, const StorageFileLocation &location,
int size, int size,
Data::FileOrigin origin); Data::FileOrigin origin);
@ -61,7 +61,7 @@ private:
void cancelForOffset(int offset); void cancelForOffset(int offset);
void changeRequestedAmount(int index, int amount); void changeRequestedAmount(int index, int amount);
const not_null<Storage::Downloader*> _owner; const not_null<Storage::DownloadManager*> _owner;
// _location can be changed with an updated file_reference. // _location can be changed with an updated file_reference.
StorageFileLocation _location; StorageFileLocation _location;

160
Telegram/SourceFiles/storage/file_download.cpp
View File

@ -45,7 +45,7 @@ constexpr auto kPartSize = 128 * 1024;
} // namespace } // namespace
void Downloader::Queue::enqueue(not_null<FileLoader*> loader) { void DownloadManager::Queue::enqueue(not_null<Downloader*> loader) {
const auto i = ranges::find(_loaders, loader); const auto i = ranges::find(_loaders, loader);
if (i != end(_loaders)) { if (i != end(_loaders)) {
return; return;
@ -56,14 +56,14 @@ void Downloader::Queue::enqueue(not_null<FileLoader*> loader) {
end(_previousGeneration)); end(_previousGeneration));
} }
void Downloader::Queue::remove(not_null<FileLoader*> loader) { void DownloadManager::Queue::remove(not_null<Downloader*> loader) {
_loaders.erase(ranges::remove(_loaders, loader), end(_loaders)); _loaders.erase(ranges::remove(_loaders, loader), end(_loaders));
_previousGeneration.erase( _previousGeneration.erase(
ranges::remove(_previousGeneration, loader), ranges::remove(_previousGeneration, loader),
end(_previousGeneration)); end(_previousGeneration));
} }
void Downloader::Queue::resetGeneration() { void DownloadManager::Queue::resetGeneration() {
if (!_previousGeneration.empty()) { if (!_previousGeneration.empty()) {
_loaders.reserve(_loaders.size() + _previousGeneration.size()); _loaders.reserve(_loaders.size() + _previousGeneration.size());
std::copy( std::copy(
@ -75,22 +75,22 @@ void Downloader::Queue::resetGeneration() {
std::swap(_loaders, _previousGeneration); std::swap(_loaders, _previousGeneration);
} }
FileLoader *Downloader::Queue::nextLoader() const { Downloader *DownloadManager::Queue::nextLoader() const {
auto &&all = ranges::view::concat(_loaders, _previousGeneration); auto &&all = ranges::view::concat(_loaders, _previousGeneration);
const auto i = ranges::find(all, true, &FileLoader::readyToRequest); const auto i = ranges::find(all, true, &FileLoader::readyToRequest);
return (i != all.end()) ? i->get() : nullptr; return (i != all.end()) ? i->get() : nullptr;
} }
Downloader::Downloader(not_null<ApiWrap*> api) DownloadManager::DownloadManager(not_null<ApiWrap*> api)
: _api(api) : _api(api)
, _killDownloadSessionsTimer([=] { killDownloadSessions(); }) { , _killDownloadSessionsTimer([=] { killDownloadSessions(); }) {
} }
Downloader::~Downloader() { DownloadManager::~DownloadManager() {
killDownloadSessions(); killDownloadSessions();
} }
void Downloader::enqueue(not_null<FileLoader*> loader) { void DownloadManager::enqueue(not_null<Downloader*> loader) {
const auto dcId = loader->dcId(); const auto dcId = loader->dcId();
(dcId ? _mtprotoLoaders[dcId] : _webLoaders).enqueue(loader); (dcId ? _mtprotoLoaders[dcId] : _webLoaders).enqueue(loader);
if (!_resettingGeneration) { if (!_resettingGeneration) {
@ -102,13 +102,13 @@ void Downloader::enqueue(not_null<FileLoader*> loader) {
checkSendNext(); checkSendNext();
} }
void Downloader::remove(not_null<FileLoader*> loader) { void DownloadManager::remove(not_null<Downloader*> loader) {
const auto dcId = loader->dcId(); const auto dcId = loader->dcId();
(dcId ? _mtprotoLoaders[dcId] : _webLoaders).remove(loader); (dcId ? _mtprotoLoaders[dcId] : _webLoaders).remove(loader);
crl::on_main(&_api->session(), [=] { checkSendNext(); }); crl::on_main(&_api->session(), [=] { checkSendNext(); });
} }
void Downloader::resetGeneration() { void DownloadManager::resetGeneration() {
_resettingGeneration = false; _resettingGeneration = false;
for (auto &[dcId, queue] : _mtprotoLoaders) { for (auto &[dcId, queue] : _mtprotoLoaders) {
queue.resetGeneration(); queue.resetGeneration();
@ -116,7 +116,7 @@ void Downloader::resetGeneration() {
_webLoaders.resetGeneration(); _webLoaders.resetGeneration();
} }
void Downloader::checkSendNext() { void DownloadManager::checkSendNext() {
for (auto &[dcId, queue] : _mtprotoLoaders) { for (auto &[dcId, queue] : _mtprotoLoaders) {
const auto bestIndex = [&] { const auto bestIndex = [&] {
const auto i = _requestedBytesAmount.find(dcId); const auto i = _requestedBytesAmount.find(dcId);
@ -143,7 +143,7 @@ void Downloader::checkSendNext() {
} }
} }
void Downloader::requestedAmountIncrement( void DownloadManager::requestedAmountIncrement(
MTP::DcId dcId, MTP::DcId dcId,
int index, int index,
int amount) { int amount) {
@ -166,14 +166,14 @@ void Downloader::requestedAmountIncrement(
} }
} }
int Downloader::chooseDcIndexForRequest(MTP::DcId dcId) { int DownloadManager::chooseDcIndexForRequest(MTP::DcId dcId) {
const auto i = _requestedBytesAmount.find(dcId); const auto i = _requestedBytesAmount.find(dcId);
return (i != end(_requestedBytesAmount)) return (i != end(_requestedBytesAmount))
? (ranges::min_element(i->second) - begin(i->second)) ? (ranges::min_element(i->second) - begin(i->second))
: 0; : 0;
} }
void Downloader::killDownloadSessionsStart(MTP::DcId dcId) { void DownloadManager::killDownloadSessionsStart(MTP::DcId dcId) {
if (!_killDownloadSessionTimes.contains(dcId)) { if (!_killDownloadSessionTimes.contains(dcId)) {
_killDownloadSessionTimes.emplace( _killDownloadSessionTimes.emplace(
dcId, dcId,
@ -184,7 +184,7 @@ void Downloader::killDownloadSessionsStart(MTP::DcId dcId) {
} }
} }
void Downloader::killDownloadSessionsStop(MTP::DcId dcId) { void DownloadManager::killDownloadSessionsStop(MTP::DcId dcId) {
_killDownloadSessionTimes.erase(dcId); _killDownloadSessionTimes.erase(dcId);
if (_killDownloadSessionTimes.empty() if (_killDownloadSessionTimes.empty()
&& _killDownloadSessionsTimer.isActive()) { && _killDownloadSessionsTimer.isActive()) {
@ -192,7 +192,7 @@ void Downloader::killDownloadSessionsStop(MTP::DcId dcId) {
} }
} }
void Downloader::killDownloadSessions() { void DownloadManager::killDownloadSessions() {
const auto now = crl::now(); const auto now = crl::now();
auto left = kKillSessionTimeout; auto left = kKillSessionTimeout;
for (auto i = _killDownloadSessionTimes.begin(); i != _killDownloadSessionTimes.end(); ) { for (auto i = _killDownloadSessionTimes.begin(); i != _killDownloadSessionTimes.end(); ) {
@ -669,8 +669,7 @@ void mtpFileLoader::refreshFileReferenceFrom(
cancel(true); cancel(true);
return; return;
} }
const auto offset = finishSentRequestGetOffset(requestId); makeRequest(finishSentRequest(requestId));
makeRequest(offset);
} }
bool mtpFileLoader::readyToRequest() const { bool mtpFileLoader::readyToRequest() const {
@ -683,28 +682,17 @@ bool mtpFileLoader::readyToRequest() const {
void mtpFileLoader::loadPart(int dcIndex) { void mtpFileLoader::loadPart(int dcIndex) {
Expects(readyToRequest()); Expects(readyToRequest());
makeRequest(_nextRequestOffset, dcIndex); makeRequest({ _nextRequestOffset, dcIndex });
_nextRequestOffset += Storage::kPartSize; _nextRequestOffset += Storage::kPartSize;
} }
mtpFileLoader::RequestData mtpFileLoader::prepareRequest(
int offset,
int dcIndex) const {
auto result = RequestData();
result.dcId = _cdnDcId ? _cdnDcId : dcId();
result.dcIndex = dcIndex;
result.offset = offset;
return result;
}
mtpRequestId mtpFileLoader::sendRequest(const RequestData &requestData) { mtpRequestId mtpFileLoader::sendRequest(const RequestData &requestData) {
const auto offset = requestData.offset; const auto offset = requestData.offset;
const auto limit = Storage::kPartSize; const auto limit = Storage::kPartSize;
const auto shiftedDcId = MTP::downloadDcId( const auto shiftedDcId = MTP::downloadDcId(
requestData.dcId, _cdnDcId ? _cdnDcId : dcId(),
requestData.dcIndex); requestData.dcIndex);
if (_cdnDcId) { if (_cdnDcId) {
Assert(requestData.dcId == _cdnDcId);
return MTP::send( return MTP::send(
MTPupload_GetCdnFile( MTPupload_GetCdnFile(
MTP_bytes(_cdnToken), MTP_bytes(_cdnToken),
@ -761,17 +749,12 @@ mtpRequestId mtpFileLoader::sendRequest(const RequestData &requestData) {
}); });
} }
void mtpFileLoader::makeRequest(int offset, int dcIndex) { void mtpFileLoader::makeRequest(const RequestData &requestData) {
Expects(!_finished); Expects(!_finished);
auto requestData = prepareRequest(offset, dcIndex);
placeSentRequest(sendRequest(requestData), requestData); placeSentRequest(sendRequest(requestData), requestData);
} }
void mtpFileLoader::makeRequest(int offset) {
makeRequest(offset, _downloader->chooseDcIndexForRequest(dcId()));
}
void mtpFileLoader::requestMoreCdnFileHashes() { void mtpFileLoader::requestMoreCdnFileHashes() {
Expects(!_finished); Expects(!_finished);
@ -779,18 +762,14 @@ void mtpFileLoader::requestMoreCdnFileHashes() {
return; return;
} }
auto offset = _cdnUncheckedParts.cbegin()->first; const auto requestData = _cdnUncheckedParts.cbegin()->first;
auto requestData = RequestData(); const auto shiftedDcId = MTP::downloadDcId(
requestData.dcId = dcId(); dcId(),
requestData.dcIndex = 0;
requestData.offset = offset;
auto shiftedDcId = MTP::downloadDcId(
requestData.dcId,
requestData.dcIndex); requestData.dcIndex);
auto requestId = _cdnHashesRequestId = MTP::send( const auto requestId = _cdnHashesRequestId = MTP::send(
MTPupload_GetCdnFileHashes( MTPupload_GetCdnFileHashes(
MTP_bytes(_cdnToken), MTP_bytes(_cdnToken),
MTP_int(offset)), MTP_int(requestData.offset)),
rpcDone(&mtpFileLoader::getCdnFileHashesDone), rpcDone(&mtpFileLoader::getCdnFileHashesDone),
rpcFail(&mtpFileLoader::cdnPartFailed), rpcFail(&mtpFileLoader::cdnPartFailed),
shiftedDcId); shiftedDcId);
@ -801,21 +780,20 @@ void mtpFileLoader::normalPartLoaded(
const MTPupload_File &result, const MTPupload_File &result,
mtpRequestId requestId) { mtpRequestId requestId) {
Expects(!_finished); Expects(!_finished);
Expects(result.type() == mtpc_upload_fileCdnRedirect || result.type() == mtpc_upload_file);
auto offset = finishSentRequestGetOffset(requestId); const auto requestData = finishSentRequest(requestId);
if (result.type() == mtpc_upload_fileCdnRedirect) { result.match([&](const MTPDupload_fileCdnRedirect &data) {
return switchToCDN(offset, result.c_upload_fileCdnRedirect()); switchToCDN(requestData, data);
} }, [&](const MTPDupload_file &data) {
auto buffer = bytes::make_span(result.c_upload_file().vbytes().v); partLoaded(requestData.offset, bytes::make_span(data.vbytes().v));
return partLoaded(offset, buffer); });
} }
void mtpFileLoader::webPartLoaded( void mtpFileLoader::webPartLoaded(
const MTPupload_WebFile &result, const MTPupload_WebFile &result,
mtpRequestId requestId) { mtpRequestId requestId) {
result.match([&](const MTPDupload_webFile &data) { result.match([&](const MTPDupload_webFile &data) {
const auto offset = finishSentRequestGetOffset(requestId); const auto requestData = finishSentRequest(requestId);
if (!_size) { if (!_size) {
_size = data.vsize().v; _size = data.vsize().v;
} else if (data.vsize().v != _size) { } else if (data.vsize().v != _size) {
@ -826,21 +804,17 @@ void mtpFileLoader::webPartLoaded(
cancel(true); cancel(true);
return; return;
} }
partLoaded(offset, bytes::make_span(data.vbytes().v)); partLoaded(requestData.offset, bytes::make_span(data.vbytes().v));
}); });
} }
void mtpFileLoader::cdnPartLoaded(const MTPupload_CdnFile &result, mtpRequestId requestId) { void mtpFileLoader::cdnPartLoaded(const MTPupload_CdnFile &result, mtpRequestId requestId) {
Expects(!_finished); Expects(!_finished);
const auto offset = finishSentRequestGetOffset(requestId); const auto requestData = finishSentRequest(requestId);
result.match([&](const MTPDupload_cdnFileReuploadNeeded &data) { result.match([&](const MTPDupload_cdnFileReuploadNeeded &data) {
auto requestData = RequestData();
requestData.dcId = dcId();
requestData.dcIndex = 0;
requestData.offset = offset;
const auto shiftedDcId = MTP::downloadDcId( const auto shiftedDcId = MTP::downloadDcId(
requestData.dcId, dcId(),
requestData.dcIndex); requestData.dcIndex);
const auto requestId = MTP::send( const auto requestId = MTP::send(
MTPupload_ReuploadCdnFile( MTPupload_ReuploadCdnFile(
@ -860,7 +834,7 @@ void mtpFileLoader::cdnPartLoaded(const MTPupload_CdnFile &result, mtpRequestId
auto ivec = bytes::make_span(state.ivec); auto ivec = bytes::make_span(state.ivec);
std::copy(iv.begin(), iv.end(), ivec.begin()); std::copy(iv.begin(), iv.end(), ivec.begin());
auto counterOffset = static_cast<uint32>(offset) >> 4; auto counterOffset = static_cast<uint32>(requestData.offset) >> 4;
state.ivec[15] = static_cast<uchar>(counterOffset & 0xFF); state.ivec[15] = static_cast<uchar>(counterOffset & 0xFF);
state.ivec[14] = static_cast<uchar>((counterOffset >> 8) & 0xFF); state.ivec[14] = static_cast<uchar>((counterOffset >> 8) & 0xFF);
state.ivec[13] = static_cast<uchar>((counterOffset >> 16) & 0xFF); state.ivec[13] = static_cast<uchar>((counterOffset >> 16) & 0xFF);
@ -870,19 +844,20 @@ void mtpFileLoader::cdnPartLoaded(const MTPupload_CdnFile &result, mtpRequestId
auto buffer = bytes::make_detached_span(decryptInPlace); auto buffer = bytes::make_detached_span(decryptInPlace);
MTP::aesCtrEncrypt(buffer, key.data(), &state); MTP::aesCtrEncrypt(buffer, key.data(), &state);
switch (checkCdnFileHash(offset, buffer)) { switch (checkCdnFileHash(requestData.offset, buffer)) {
case CheckCdnHashResult::NoHash: { case CheckCdnHashResult::NoHash: {
_cdnUncheckedParts.emplace(offset, decryptInPlace); _cdnUncheckedParts.emplace(requestData, decryptInPlace);
requestMoreCdnFileHashes(); requestMoreCdnFileHashes();
} return; } return;
case CheckCdnHashResult::Invalid: { case CheckCdnHashResult::Invalid: {
LOG(("API Error: Wrong cdnFileHash for offset %1.").arg(offset)); LOG(("API Error: Wrong cdnFileHash for offset %1."
).arg(requestData.offset));
cancel(true); cancel(true);
} return; } return;
case CheckCdnHashResult::Good: { case CheckCdnHashResult::Good: {
partLoaded(offset, buffer); partLoaded(requestData.offset, buffer);
} return; } return;
} }
Unexpected("Result of checkCdnFileHash()"); Unexpected("Result of checkCdnFileHash()");
@ -907,9 +882,9 @@ mtpFileLoader::CheckCdnHashResult mtpFileLoader::checkCdnFileHash(
void mtpFileLoader::reuploadDone( void mtpFileLoader::reuploadDone(
const MTPVector<MTPFileHash> &result, const MTPVector<MTPFileHash> &result,
mtpRequestId requestId) { mtpRequestId requestId) {
auto offset = finishSentRequestGetOffset(requestId); const auto requestData = finishSentRequest(requestId);
addCdnHashes(result.v); addCdnHashes(result.v);
makeRequest(offset); makeRequest(requestData);
} }
void mtpFileLoader::getCdnFileHashesDone( void mtpFileLoader::getCdnFileHashesDone(
@ -920,27 +895,28 @@ void mtpFileLoader::getCdnFileHashesDone(
_cdnHashesRequestId = 0; _cdnHashesRequestId = 0;
const auto offset = finishSentRequestGetOffset(requestId); const auto requestData = finishSentRequest(requestId);
addCdnHashes(result.v); addCdnHashes(result.v);
auto someMoreChecked = false; auto someMoreChecked = false;
for (auto i = _cdnUncheckedParts.begin(); i != _cdnUncheckedParts.cend();) { for (auto i = _cdnUncheckedParts.begin(); i != _cdnUncheckedParts.cend();) {
const auto uncheckedOffset = i->first; const auto uncheckedData = i->first;
const auto uncheckedBytes = bytes::make_span(i->second); const auto uncheckedBytes = bytes::make_span(i->second);
switch (checkCdnFileHash(uncheckedOffset, uncheckedBytes)) { switch (checkCdnFileHash(uncheckedData.offset, uncheckedBytes)) {
case CheckCdnHashResult::NoHash: { case CheckCdnHashResult::NoHash: {
++i; ++i;
} break; } break;
case CheckCdnHashResult::Invalid: { case CheckCdnHashResult::Invalid: {
LOG(("API Error: Wrong cdnFileHash for offset %1.").arg(offset)); LOG(("API Error: Wrong cdnFileHash for offset %1."
).arg(uncheckedData.offset));
cancel(true); cancel(true);
return; return;
} break; } break;
case CheckCdnHashResult::Good: { case CheckCdnHashResult::Good: {
someMoreChecked = true; someMoreChecked = true;
const auto goodOffset = uncheckedOffset; const auto goodOffset = uncheckedData.offset;
const auto goodBytes = std::move(i->second); const auto goodBytes = std::move(i->second);
const auto weak = QPointer<mtpFileLoader>(this); const auto weak = QPointer<mtpFileLoader>(this);
i = _cdnUncheckedParts.erase(i); i = _cdnUncheckedParts.erase(i);
@ -967,7 +943,7 @@ void mtpFileLoader::getCdnFileHashesDone(
LOG(("API Error: " LOG(("API Error: "
"Could not find cdnFileHash for offset %1 " "Could not find cdnFileHash for offset %1 "
"after getCdnFileHashes request." "after getCdnFileHashes request."
).arg(offset)); ).arg(requestData.offset));
cancel(true); cancel(true);
} }
@ -977,24 +953,25 @@ void mtpFileLoader::placeSentRequest(
Expects(!_finished); Expects(!_finished);
_downloader->requestedAmountIncrement( _downloader->requestedAmountIncrement(
requestData.dcId, dcId(),
requestData.dcIndex, requestData.dcIndex,
Storage::kPartSize); Storage::kPartSize);
_sentRequests.emplace(requestId, requestData); _sentRequests.emplace(requestId, requestData);
} }
int mtpFileLoader::finishSentRequestGetOffset(mtpRequestId requestId) { auto mtpFileLoader::finishSentRequest(mtpRequestId requestId)
-> RequestData {
auto it = _sentRequests.find(requestId); auto it = _sentRequests.find(requestId);
Assert(it != _sentRequests.cend()); Assert(it != _sentRequests.cend());
auto requestData = it->second; const auto result = it->second;
_downloader->requestedAmountIncrement( _downloader->requestedAmountIncrement(
requestData.dcId, dcId(),
requestData.dcIndex, result.dcIndex,
-Storage::kPartSize); -Storage::kPartSize);
_sentRequests.erase(it); _sentRequests.erase(it);
return requestData.offset; return result;
} }
bool mtpFileLoader::feedPart(int offset, bytes::const_span buffer) { bool mtpFileLoader::feedPart(int offset, bytes::const_span buffer) {
@ -1061,9 +1038,9 @@ bool mtpFileLoader::cdnPartFailed(
} }
if (error.type() == qstr("FILE_TOKEN_INVALID") if (error.type() == qstr("FILE_TOKEN_INVALID")
|| error.type() == qstr("REQUEST_TOKEN_INVALID")) { || error.type() == qstr("REQUEST_TOKEN_INVALID")) {
auto offset = finishSentRequestGetOffset(requestId); const auto requestData = finishSentRequest(requestId);
changeCDNParams( changeCDNParams(
offset, requestData,
0, 0,
QByteArray(), QByteArray(),
QByteArray(), QByteArray(),
@ -1078,15 +1055,15 @@ void mtpFileLoader::cancelRequests() {
while (!_sentRequests.empty()) { while (!_sentRequests.empty()) {
auto requestId = _sentRequests.begin()->first; auto requestId = _sentRequests.begin()->first;
MTP::cancel(requestId); MTP::cancel(requestId);
finishSentRequestGetOffset(requestId); [[maybe_unused]] const auto data = finishSentRequest(requestId);
} }
} }
void mtpFileLoader::switchToCDN( void mtpFileLoader::switchToCDN(
int offset, const RequestData &requestData,
const MTPDupload_fileCdnRedirect &redirect) { const MTPDupload_fileCdnRedirect &redirect) {
changeCDNParams( changeCDNParams(
offset, requestData,
redirect.vdc_id().v, redirect.vdc_id().v,
redirect.vfile_token().v, redirect.vfile_token().v,
redirect.vencryption_key().v, redirect.vencryption_key().v,
@ -1105,7 +1082,7 @@ void mtpFileLoader::addCdnHashes(const QVector<MTPFileHash> &hashes) {
} }
void mtpFileLoader::changeCDNParams( void mtpFileLoader::changeCDNParams(
int offset, const RequestData &requestData,
MTP::DcId dcId, MTP::DcId dcId,
const QByteArray &token, const QByteArray &token,
const QByteArray &encryptionKey, const QByteArray &encryptionKey,
@ -1130,19 +1107,18 @@ void mtpFileLoader::changeCDNParams(
addCdnHashes(hashes); addCdnHashes(hashes);
if (resendAllRequests && !_sentRequests.empty()) { if (resendAllRequests && !_sentRequests.empty()) {
auto resendOffsets = std::vector<int>(); auto resendRequests = std::vector<RequestData>();
resendOffsets.reserve(_sentRequests.size()); resendRequests.reserve(_sentRequests.size());
while (!_sentRequests.empty()) { while (!_sentRequests.empty()) {
auto requestId = _sentRequests.begin()->first; auto requestId = _sentRequests.begin()->first;
MTP::cancel(requestId); MTP::cancel(requestId);
auto resendOffset = finishSentRequestGetOffset(requestId); resendRequests.push_back(finishSentRequest(requestId));
resendOffsets.push_back(resendOffset);
} }
for (auto resendOffset : resendOffsets) { for (const auto &requestData : resendRequests) {
makeRequest(resendOffset); makeRequest(requestData);
} }
} }
makeRequest(offset); makeRequest(requestData);
} }
Storage::Cache::Key mtpFileLoader::cacheKey() const { Storage::Cache::Key mtpFileLoader::cacheKey() const {

76
Telegram/SourceFiles/storage/file_download.h
View File

@ -16,7 +16,6 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
#include <QtNetwork/QNetworkReply> #include <QtNetwork/QNetworkReply>
class ApiWrap; class ApiWrap;
class FileLoader;
namespace Main { namespace Main {
class Session; class Session;
@ -36,17 +35,27 @@ constexpr auto kMaxWallPaperInMemory = kMaxFileInMemory;
constexpr auto kMaxAnimationInMemory = kMaxFileInMemory; // 10 MB gif and mp4 animations held in memory while playing constexpr auto kMaxAnimationInMemory = kMaxFileInMemory; // 10 MB gif and mp4 animations held in memory while playing
constexpr auto kMaxWallPaperDimension = 4096; // 4096x4096 is max area. constexpr auto kMaxWallPaperDimension = 4096; // 4096x4096 is max area.
class Downloader final : public base::has_weak_ptr { class Downloader {
public: public:
explicit Downloader(not_null<ApiWrap*> api); virtual ~Downloader() = default;
~Downloader();
[[nodiscard]] virtual MTP::DcId dcId() const = 0;
[[nodiscard]] virtual bool readyToRequest() const = 0;
[[nodiscard]] virtual void loadPart(int dcIndex) = 0;
};
class DownloadManager final : public base::has_weak_ptr {
public:
explicit DownloadManager(not_null<ApiWrap*> api);
~DownloadManager();
[[nodiscard]] ApiWrap &api() const { [[nodiscard]] ApiWrap &api() const {
return *_api; return *_api;
} }
void enqueue(not_null<FileLoader*> loader); void enqueue(not_null<Downloader*> loader);
void remove(not_null<FileLoader*> loader); void remove(not_null<Downloader*> loader);
[[nodiscard]] base::Observable<void> &taskFinished() { [[nodiscard]] base::Observable<void> &taskFinished() {
return _taskFinishedObservable; return _taskFinishedObservable;
@ -59,14 +68,14 @@ public:
private: private:
class Queue final { class Queue final {
public: public:
void enqueue(not_null<FileLoader*> loader); void enqueue(not_null<Downloader*> loader);
void remove(not_null<FileLoader*> loader); void remove(not_null<Downloader*> loader);
void resetGeneration(); void resetGeneration();
[[nodiscard]] FileLoader *nextLoader() const; [[nodiscard]] Downloader *nextLoader() const;
private: private:
std::vector<not_null<FileLoader*>> _loaders; std::vector<not_null<Downloader*>> _loaders;
std::vector<not_null<FileLoader*>> _previousGeneration; std::vector<not_null<Downloader*>> _previousGeneration;
}; };
@ -105,7 +114,7 @@ struct StorageImageSaved {
}; };
class FileLoader : public QObject { class FileLoader : public QObject, public Storage::Downloader {
Q_OBJECT Q_OBJECT
public: public:
@ -171,7 +180,7 @@ signals:
void failed(FileLoader *loader, bool started); void failed(FileLoader *loader, bool started);
protected: protected:
friend class Storage::Downloader; friend class Storage::DownloadManager;
enum class LocalStatus { enum class LocalStatus {
NotTried, NotTried,
@ -180,7 +189,7 @@ protected:
Loaded, Loaded,
}; };
[[nodiscard]] MTP::DcId dcId() const { MTP::DcId dcId() const override {
return _dcId; return _dcId;
} }
@ -195,15 +204,13 @@ protected:
void cancel(bool failed); void cancel(bool failed);
void notifyAboutProgress(); void notifyAboutProgress();
[[nodiscard]] virtual bool readyToRequest() const = 0;
virtual void loadPart(int dcIndex) = 0;
bool writeResultPart(int offset, bytes::const_span buffer); bool writeResultPart(int offset, bytes::const_span buffer);
bool finalizeResult(); bool finalizeResult();
[[nodiscard]] QByteArray readLoadedPartBack(int offset, int size); [[nodiscard]] QByteArray readLoadedPartBack(int offset, int size);
const MTP::DcId _dcId = 0; const MTP::DcId _dcId = 0;
const not_null<Storage::Downloader*> _downloader; const not_null<Storage::DownloadManager*> _downloader;
bool _autoLoading = false; bool _autoLoading = false;
uint8 _cacheTag = 0; uint8 _cacheTag = 0;
@ -272,12 +279,15 @@ public:
~mtpFileLoader(); ~mtpFileLoader();
private: private:
friend class Downloader; friend class DownloadManager;
struct RequestData { struct RequestData {
MTP::DcId dcId = 0;
int dcIndex = 0;
int offset = 0; int offset = 0;
int dcIndex = 0;
inline bool operator<(const RequestData &other) const {
return offset < other.offset;
}
}; };
struct CdnFileHash { struct CdnFileHash {
CdnFileHash(int limit, QByteArray hash) : limit(limit), hash(hash) { CdnFileHash(int limit, QByteArray hash) : limit(limit), hash(hash) {
@ -289,9 +299,7 @@ private:
std::optional<MediaKey> fileLocationKey() const override; std::optional<MediaKey> fileLocationKey() const override;
void cancelRequests() override; void cancelRequests() override;
[[nodiscard]] RequestData prepareRequest(int offset, int dcIndex) const; void makeRequest(const RequestData &requestData);
void makeRequest(int offset, int dcIndex);
void makeRequest(int offset);
bool readyToRequest() const override; bool readyToRequest() const override;
void loadPart(int dcIndex) override; void loadPart(int dcIndex) override;
@ -310,11 +318,21 @@ private:
bool cdnPartFailed(const RPCError &error, mtpRequestId requestId); bool cdnPartFailed(const RPCError &error, mtpRequestId requestId);
mtpRequestId sendRequest(const RequestData &requestData); mtpRequestId sendRequest(const RequestData &requestData);
void placeSentRequest(mtpRequestId requestId, const RequestData &requestData); void placeSentRequest(
int finishSentRequestGetOffset(mtpRequestId requestId); mtpRequestId requestId,
void switchToCDN(int offset, const MTPDupload_fileCdnRedirect &redirect); const RequestData &requestData);
[[nodiscard]] RequestData finishSentRequest(mtpRequestId requestId);
void switchToCDN(
const RequestData &requestData,
const MTPDupload_fileCdnRedirect &redirect);
void addCdnHashes(const QVector<MTPFileHash> &hashes); void addCdnHashes(const QVector<MTPFileHash> &hashes);
void changeCDNParams(int offset, MTP::DcId dcId, const QByteArray &token, const QByteArray &encryptionKey, const QByteArray &encryptionIV, const QVector<MTPFileHash> &hashes); void changeCDNParams(
const RequestData &requestData,
MTP::DcId dcId,
const QByteArray &token,
const QByteArray &encryptionKey,
const QByteArray &encryptionIV,
const QVector<MTPFileHash> &hashes);
enum class CheckCdnHashResult { enum class CheckCdnHashResult {
NoHash, NoHash,
@ -338,8 +356,8 @@ private:
QByteArray _cdnToken; QByteArray _cdnToken;
QByteArray _cdnEncryptionKey; QByteArray _cdnEncryptionKey;
QByteArray _cdnEncryptionIV; QByteArray _cdnEncryptionIV;
std::map<int, CdnFileHash> _cdnFileHashes; base::flat_map<int, CdnFileHash> _cdnFileHashes;
std::map<int, QByteArray> _cdnUncheckedParts; base::flat_map<RequestData, QByteArray> _cdnUncheckedParts;
mtpRequestId _cdnHashesRequestId = 0; mtpRequestId _cdnHashesRequestId = 0;
}; };