Use vector queues in mtproto file downloader.

Telegram/SourceFiles/boxes/peer_list_box.cpp

@@ -1287,7 +1287,6 @@ void PeerListContent::loadProfilePhotos() {
 
 	auto yFrom = _visibleTop;
 	auto yTo = _visibleBottom + (_visibleBottom - _visibleTop) * PreloadHeightsCount;
-	_controller->session().downloader().clearPriorities();
 
 	if (yTo < 0) return;
 	if (yFrom < 0) yFrom = 0;

Telegram/SourceFiles/boxes/share_box.cpp

@@ -722,7 +722,6 @@ void ShareBox::Inner::loadProfilePhotos(int yFrom) {
 	yFrom *= _columnCount;
 	yTo *= _columnCount;
 
-	_navigation->session().downloader().clearPriorities();
 	if (_filter.isEmpty()) {
 		if (!_chatsIndexed->empty()) {
 			auto i = _chatsIndexed->cfind(yFrom, _rowHeight);

Telegram/SourceFiles/dialogs/dialogs_inner_widget.cpp

@@ -2464,7 +2464,6 @@ void InnerWidget::loadPeerPhotos() {
 
 	auto yFrom = _visibleTop;
 	auto yTo = _visibleTop + (_visibleBottom - _visibleTop) * (PreloadHeightsCount + 1);
-	session().downloader().clearPriorities();
 	if (_state == WidgetState::Default) {
 		auto otherStart = shownDialogs()->size() * st::dialogsRowHeight;
 		if (yFrom < otherStart) {

Telegram/SourceFiles/history/history_widget.cpp

@@ -1798,8 +1798,6 @@ void HistoryWidget::showHistory(
 	_nonEmptySelection = false;
 
 	if (_peer) {
-		session().downloader().clearPriorities();
-
 		_history = _peer->owner().history(_peer);
 		_migrated = _history->migrateFrom();
 		if (_migrated

Telegram/SourceFiles/media/view/media_view_overlay_widget.cpp

@@ -1803,7 +1803,6 @@ void OverlayWidget::displayPhoto(not_null<PhotoData*> photo, HistoryItem *item)
 
 	_zoom = 0;
 	_zoomToScreen = 0;
-	Auth().downloader().clearPriorities();
 	_blurred = true;
 	_current = QPixmap();
 	_down = OverNone;

Telegram/SourceFiles/storage/file_download.cpp

@@ -32,6 +32,8 @@ constexpr auto kKillSessionTimeout = 15 * crl::time(1000);
 // Max 16 file parts downloaded at the same time, 128 KB each.
 constexpr auto kMaxFileQueries = 16;
 
+constexpr auto kMaxWaitedInConnection = 512 * 1024;
+
 // Max 8 http[s] files downloaded at the same time.
 constexpr auto kMaxWebFileQueries = 8;
 
@@ -43,17 +45,108 @@ constexpr auto kPartSize = 128 * 1024;
 
 } // namespace
 
+void Downloader::Queue::enqueue(not_null<FileLoader*> loader) {
+	const auto i = ranges::find(_loaders, loader);
+	if (i != end(_loaders)) {
+		return;
+	}
+	_loaders.push_back(loader);
+	_previousGeneration.erase(
+		ranges::remove(_previousGeneration, loader),
+		end(_previousGeneration));
+}
+
+void Downloader::Queue::remove(not_null<FileLoader*> loader) {
+	_loaders.erase(ranges::remove(_loaders, loader), end(_loaders));
+	_previousGeneration.erase(
+		ranges::remove(_previousGeneration, loader),
+		end(_previousGeneration));
+}
+
+void Downloader::Queue::resetGeneration() {
+	if (!_previousGeneration.empty()) {
+		_loaders.reserve(_loaders.size() + _previousGeneration.size());
+		std::copy(
+			begin(_previousGeneration),
+			end(_previousGeneration),
+			std::back_inserter(_loaders));
+		_previousGeneration.clear();
+	}
+	std::swap(_loaders, _previousGeneration);
+}
+
+FileLoader *Downloader::Queue::nextLoader() const {
+	auto &&all = ranges::view::concat(_loaders, _previousGeneration);
+	const auto i = ranges::find(all, true, &FileLoader::readyToRequest);
+	return (i != all.end()) ? i->get() : nullptr;
+}
+
 Downloader::Downloader(not_null<ApiWrap*> api)
 : _api(api)
-, _killDownloadSessionsTimer([=] { killDownloadSessions(); })
+, _killDownloadSessionsTimer([=] { killDownloadSessions(); }) {
-, _queueForWeb(kMaxWebFileQueries) {
 }
 
-void Downloader::clearPriorities() {
+Downloader::~Downloader() {
-	++_priority;
+	killDownloadSessions();
 }
 
-void Downloader::requestedAmountIncrement(MTP::DcId dcId, int index, int amount) {
+void Downloader::enqueue(not_null<FileLoader*> loader) {
+	const auto dcId = loader->dcId();
+	(dcId ? _mtprotoLoaders[dcId] : _webLoaders).enqueue(loader);
+	if (!_resettingGeneration) {
+		_resettingGeneration = true;
+		crl::on_main(this, [=] {
+			resetGeneration();
+		});
+	}
+	checkSendNext();
+}
+
+void Downloader::remove(not_null<FileLoader*> loader) {
+	const auto dcId = loader->dcId();
+	(dcId ? _mtprotoLoaders[dcId] : _webLoaders).remove(loader);
+	crl::on_main(&_api->session(), [=] { checkSendNext(); });
+}
+
+void Downloader::resetGeneration() {
+	_resettingGeneration = false;
+	for (auto &[dcId, queue] : _mtprotoLoaders) {
+		queue.resetGeneration();
+	}
+	_webLoaders.resetGeneration();
+}
+
+void Downloader::checkSendNext() {
+	for (auto &[dcId, queue] : _mtprotoLoaders) {
+		const auto bestIndex = [&] {
+			const auto i = _requestedBytesAmount.find(dcId);
+			if (i == end(_requestedBytesAmount)) {
+				return 0;
+			}
+			const auto j = ranges::min_element(i->second);
+			const auto inConnection = *j;
+			return (inConnection + kPartSize <= kMaxWaitedInConnection)
+				? (j - begin(i->second))
+				: -1;
+		}();
+		if (bestIndex < 0) {
+			continue;
+		}
+		if (const auto loader = queue.nextLoader()) {
+			loader->loadPart(bestIndex);
+		}
+	}
+	if (_requestedBytesAmount[0][0] < kMaxWebFileQueries) {
+		if (const auto loader = _webLoaders.nextLoader()) {
+			loader->loadPart(0);
+		}
+	}
+}
+
+void Downloader::requestedAmountIncrement(
+		MTP::DcId dcId,
+		int index,
+		int amount) {
 	Expects(index >= 0 && index < MTP::kDownloadSessionsCount);
 
 	using namespace rpl::mappers;
@@ -63,6 +156,9 @@ void Downloader::requestedAmountIncrement(MTP::DcId dcId, int index, int amount)
 		it = _requestedBytesAmount.emplace(dcId, RequestedInDc { { 0 } }).first;
 	}
 	it->second[index] += amount;
+	if (!dcId) {
+		return; // webLoaders.
+	}
 	if (amount > 0) {
 		killDownloadSessionsStop(dcId);
 	} else if (ranges::find_if(it->second, _1 > 0) == end(it->second)) {
@@ -70,6 +166,13 @@ void Downloader::requestedAmountIncrement(MTP::DcId dcId, int index, int amount)
 	}
 }
 
+int Downloader::chooseDcIndexForRequest(MTP::DcId dcId) {
+	const auto i = _requestedBytesAmount.find(dcId);
+	return (i != end(_requestedBytesAmount))
+		? (ranges::min_element(i->second) - begin(i->second))
+		: 0;
+}
+
 void Downloader::killDownloadSessionsStart(MTP::DcId dcId) {
 	if (!_killDownloadSessionTimes.contains(dcId)) {
 		_killDownloadSessionTimes.emplace(
@@ -110,35 +213,6 @@ void Downloader::killDownloadSessions() {
 	}
 }
 
-int Downloader::chooseDcIndexForRequest(MTP::DcId dcId) const {
-	auto result = 0;
-	auto it = _requestedBytesAmount.find(dcId);
-	if (it != _requestedBytesAmount.cend()) {
-		for (auto i = 1; i != MTP::kDownloadSessionsCount; ++i) {
-			if (it->second[i] < it->second[result]) {
-				result = i;
-			}
-		}
-	}
-	return result;
-}
-
-not_null<Downloader::Queue*> Downloader::queueForDc(MTP::DcId dcId) {
-	const auto i = _queuesForDc.find(dcId);
-	const auto result = (i != end(_queuesForDc))
-		? i
-		: _queuesForDc.emplace(dcId, Queue(kMaxFileQueries)).first;
-	return &result->second;
-}
-
-not_null<Downloader::Queue*> Downloader::queueForWeb() {
-	return &_queueForWeb;
-}
-
-Downloader::~Downloader() {
-	killDownloadSessions();
-}
-
 } // namespace Storage
 
 namespace {
@@ -154,13 +228,15 @@ WebLoadMainManager *_webLoadMainManager = nullptr;
 
 FileLoader::FileLoader(
 	const QString &toFile,
+	MTP::DcId dcId,
 	int32 size,
 	LocationType locationType,
 	LoadToCacheSetting toCache,
 	LoadFromCloudSetting fromCloud,
 	bool autoLoading,
 	uint8 cacheTag)
-: _downloader(&Auth().downloader())
+: _dcId(dcId)
+, _downloader(&Auth().downloader())
 , _autoLoading(autoLoading)
 , _cacheTag(cacheTag)
 , _filename(toFile)
@@ -172,6 +248,10 @@ FileLoader::FileLoader(
 	Expects(!_filename.isEmpty() || (_size <= Storage::kMaxFileInMemory));
 }
 
+FileLoader::~FileLoader() {
+	_downloader->remove(this);
+}
+
 Main::Session &FileLoader::session() const {
 	return _downloader->api().session();
 }
@@ -257,46 +337,7 @@ void FileLoader::permitLoadFromCloud() {
 }
 
 void FileLoader::notifyAboutProgress() {
-	const auto queue = _queue;
 	emit progress(this);
-	LoadNextFromQueue(queue);
-}
-
-void FileLoader::LoadNextFromQueue(not_null<Queue*> queue) {
-	if (queue->queriesCount >= queue->queriesLimit) {
-		return;
-	}
-	for (auto i = queue->start; i;) {
-		if (i->loadPart()) {
-			if (queue->queriesCount >= queue->queriesLimit) {
-				return;
-			}
-		} else {
-			i = i->_next;
-		}
-	}
-}
-
-void FileLoader::removeFromQueue() {
-	if (!_inQueue) return;
-	if (_next) {
-		_next->_prev = _prev;
-	}
-	if (_prev) {
-		_prev->_next = _next;
-	}
-	if (_queue->end == this) {
-		_queue->end = _prev;
-	}
-	if (_queue->start == this) {
-		_queue->start = _next;
-	}
-	_next = _prev = nullptr;
-	_inQueue = false;
-}
-
-FileLoader::~FileLoader() {
-	removeFromQueue();
 }
 
 void FileLoader::localLoaded(
@@ -331,71 +372,7 @@ void FileLoader::start() {
 			return cancel(true);
 		}
 	}
-
+	_downloader->enqueue(this);
-	auto currentPriority = _downloader->currentPriority();
-	FileLoader *before = nullptr, *after = nullptr;
-	if (_inQueue && _priority == currentPriority) {
-		if (!_next || _next->_priority < currentPriority) return startLoading();
-		after = _next;
-		while (after->_next && after->_next->_priority == currentPriority) {
-			after = after->_next;
-		}
-	} else {
-		_priority = currentPriority;
-		if (_inQueue) {
-			if (_next && _next->_priority == currentPriority) {
-				after = _next;
-			} else if (_prev && _prev->_priority < currentPriority) {
-				before = _prev;
-				while (before->_prev && before->_prev->_priority < currentPriority) {
-					before = before->_prev;
-				}
-			} else {
-				return startLoading();
-			}
-		} else {
-			if (_queue->start && _queue->start->_priority == currentPriority) {
-				after = _queue->start;
-			} else {
-				before = _queue->start;
-			}
-		}
-		if (after) {
-			while (after->_next && after->_next->_priority == currentPriority) {
-				after = after->_next;
-			}
-		}
-	}
-
-	removeFromQueue();
-
-	_inQueue = true;
-	if (!_queue->start) {
-		_queue->start = _queue->end = this;
-	} else if (before) {
-		if (before != _next) {
-			_prev = before->_prev;
-			_next = before;
-			_next->_prev = this;
-			if (_prev) {
-				_prev->_next = this;
-			}
-			if (_queue->start->_prev) _queue->start = _queue->start->_prev;
-		}
-	} else if (after) {
-		if (after != _prev) {
-			_next = after->_next;
-			_prev = after;
-			after->_next = this;
-			if (_next) {
-				_next->_prev = this;
-			}
-			if (_queue->end->_next) _queue->end = _queue->end->_next;
-		}
-	} else {
-		LOG(("Queue Error: _start && !before && !after"));
-	}
-	return startLoading();
 }
 
 void FileLoader::loadLocal(const Storage::Cache::Key &key) {
@@ -480,9 +457,8 @@ void FileLoader::cancel(bool fail) {
 		_file.remove();
 	}
 	_data = QByteArray();
-	removeFromQueue();
 
-	const auto queue = _queue;
+	const auto downloader = _downloader;
 	const auto sessionGuard = &session();
 	const auto weak = QPointer<FileLoader>(this);
 	if (fail) {
@@ -494,16 +470,6 @@ void FileLoader::cancel(bool fail) {
 		_filename = QString();
 		_file.setFileName(_filename);
 	}
-
-	// Current cancel() call could be made from ~Main::Session().
-	crl::on_main(sessionGuard, [=] { LoadNextFromQueue(queue); });
-}
-
-void FileLoader::startLoading() {
-	if ((_queue->queriesCount >= _queue->queriesLimit) || _finished) {
-		return;
-	}
-	loadPart();
 }
 
 int FileLoader::currentOffset() const {
@@ -594,7 +560,7 @@ bool FileLoader::finalizeResult() {
 		Platform::File::PostprocessDownloaded(
 			QFileInfo(_file).absoluteFilePath());
 	}
-	removeFromQueue();
+	_downloader->remove(this);
 
 	if (_localStatus == LocalStatus::NotFound) {
 		if (const auto key = fileLocationKey()) {
@@ -627,6 +593,7 @@ mtpFileLoader::mtpFileLoader(
 	uint8 cacheTag)
 : FileLoader(
 	to,
+	location.dcId(),
 	size,
 	type,
 	toCache,
@@ -635,7 +602,6 @@ mtpFileLoader::mtpFileLoader(
 	cacheTag)
 , _location(location)
 , _origin(origin) {
-	_queue = _downloader->queueForDc(dcId());
 }
 
 mtpFileLoader::mtpFileLoader(
@@ -646,6 +612,7 @@ mtpFileLoader::mtpFileLoader(
 	uint8 cacheTag)
 : FileLoader(
 	QString(),
+	Global::WebFileDcId(),
 	size,
 	UnknownFileLocation,
 	LoadToCacheAsWell,
@@ -653,7 +620,6 @@ mtpFileLoader::mtpFileLoader(
 	autoLoading,
 	cacheTag)
 , _location(location) {
-	_queue = _downloader->queueForDc(dcId());
 }
 
 mtpFileLoader::mtpFileLoader(
@@ -664,6 +630,7 @@ mtpFileLoader::mtpFileLoader(
 	uint8 cacheTag)
 : FileLoader(
 	QString(),
+	Global::WebFileDcId(),
 	size,
 	UnknownFileLocation,
 	LoadToCacheAsWell,
@@ -671,7 +638,10 @@ mtpFileLoader::mtpFileLoader(
 	autoLoading,
 	cacheTag)
 , _location(location) {
-	_queue = _downloader->queueForDc(dcId());
+}
+
+mtpFileLoader::~mtpFileLoader() {
+	cancelRequests();
 }
 
 Data::FileOrigin mtpFileLoader::fileOrigin() const {
@@ -703,31 +673,26 @@ void mtpFileLoader::refreshFileReferenceFrom(
 	makeRequest(offset);
 }
 
-bool mtpFileLoader::loadPart() {
+bool mtpFileLoader::readyToRequest() const {
-	if (_finished || _lastComplete || (!_sentRequests.empty() && !_size)) {
+	return !_finished
-		return false;
+		&& !_lastComplete
-	} else if (_size && _nextRequestOffset >= _size) {
+		&& (_sentRequests.empty() || _size != 0)
-		return false;
+		&& (!_size || _nextRequestOffset < _size);
-	}
+}
 
-	makeRequest(_nextRequestOffset);
+void mtpFileLoader::loadPart(int dcIndex) {
+	Expects(readyToRequest());
+
+	makeRequest(_nextRequestOffset, dcIndex);
 	_nextRequestOffset += Storage::kPartSize;
-	return true;
 }
 
-MTP::DcId mtpFileLoader::dcId() const {
+mtpFileLoader::RequestData mtpFileLoader::prepareRequest(
-	if (const auto storage = base::get_if<StorageFileLocation>(&_location)) {
+		int offset,
-		return storage->dcId();
+		int dcIndex) const {
-	}
-	return Global::WebFileDcId();
-}
-
-mtpFileLoader::RequestData mtpFileLoader::prepareRequest(int offset) const {
 	auto result = RequestData();
 	result.dcId = _cdnDcId ? _cdnDcId : dcId();
-	result.dcIndex = _size
+	result.dcIndex = dcIndex;
-		? _downloader->chooseDcIndexForRequest(result.dcId)
-		: 0;
 	result.offset = offset;
 	return result;
 }
@@ -796,13 +761,17 @@ mtpRequestId mtpFileLoader::sendRequest(const RequestData &requestData) {
 	});
 }
 
-void mtpFileLoader::makeRequest(int offset) {
+void mtpFileLoader::makeRequest(int offset, int dcIndex) {
 	Expects(!_finished);
 
-	auto requestData = prepareRequest(offset);
+	auto requestData = prepareRequest(offset, dcIndex);
 	placeSentRequest(sendRequest(requestData), requestData);
 }
 
+void mtpFileLoader::makeRequest(int offset) {
+	makeRequest(offset, _downloader->chooseDcIndexForRequest(dcId()));
+}
+
 void mtpFileLoader::requestMoreCdnFileHashes() {
 	Expects(!_finished);
 
@@ -1011,7 +980,6 @@ void mtpFileLoader::placeSentRequest(
 		requestData.dcId,
 		requestData.dcIndex,
 		Storage::kPartSize);
-	++_queue->queriesCount;
 	_sentRequests.emplace(requestId, requestData);
 }
 
@@ -1024,8 +992,6 @@ int mtpFileLoader::finishSentRequestGetOffset(mtpRequestId requestId) {
 		requestData.dcId,
 		requestData.dcIndex,
 		-Storage::kPartSize);
-
-	--_queue->queriesCount;
 	_sentRequests.erase(it);
 
 	return requestData.offset;
@@ -1196,10 +1162,6 @@ std::optional<MediaKey> mtpFileLoader::fileLocationKey() const {
 	return std::nullopt;
 }
 
-mtpFileLoader::~mtpFileLoader() {
-	cancelRequests();
-}
-
 webFileLoader::webFileLoader(
 	const QString &url,
 	const QString &to,
@@ -1209,21 +1171,28 @@ webFileLoader::webFileLoader(
 : FileLoader(
 	QString(),
 	0,
+	0,
 	UnknownFileLocation,
 	LoadToCacheAsWell,
 	fromCloud,
 	autoLoading,
 	cacheTag)
 , _url(url) {
-	_queue = _downloader->queueForWeb();
 }
 
-bool webFileLoader::loadPart() {
+webFileLoader::~webFileLoader() {
-	if (_finished
+	markAsNotSent();
-		|| _requestSent
+}
-		|| _webLoadManager == FinishedWebLoadManager) {
+
-		return false;
+bool webFileLoader::readyToRequest() const {
-	}
+	return !_finished
+		&& !_requestSent
+		&& (_webLoadManager != FinishedWebLoadManager);
+}
+
+void webFileLoader::loadPart(int dcIndex) {
+	Expects(readyToRequest());
+
 	if (!_webLoadManager) {
 		_webLoadMainManager = new WebLoadMainManager();
 
@@ -1233,9 +1202,8 @@ bool webFileLoader::loadPart() {
 		_webLoadThread->start();
 	}
 
-	_requestSent = true;
+	markAsSent();
 	_webLoadManager->append(this, _url);
-	return false;
 }
 
 int webFileLoader::currentOffset() const {
@@ -1249,6 +1217,7 @@ void webFileLoader::loadProgress(qint64 already, qint64 size) {
 }
 
 void webFileLoader::loadFinished(const QByteArray &data) {
+	markAsNotSent();
 	if (writeResultPart(0, bytes::make_span(data))) {
 		if (finalizeResult()) {
 			notifyAboutProgress();
@@ -1257,6 +1226,7 @@ void webFileLoader::loadFinished(const QByteArray &data) {
 }
 
 void webFileLoader::loadError() {
+	markAsNotSent();
 	cancel(true);
 }
 
@@ -1269,11 +1239,27 @@ std::optional<MediaKey> webFileLoader::fileLocationKey() const {
 }
 
 void webFileLoader::cancelRequests() {
-	if (!webLoadManager()) return;
+	if (!webLoadManager()) {
+		return;
+	}
 	webLoadManager()->stop(this);
+	markAsNotSent();
 }
 
-webFileLoader::~webFileLoader() {
+void webFileLoader::markAsSent() {
+	if (_requestSent) {
+		return;
+	}
+	_requestSent = true;
+	_downloader->requestedAmountIncrement(0, 0, 1);
+}
+
+void webFileLoader::markAsNotSent() {
+	if (!_requestSent) {
+		return;
+	}
+	_requestSent = false;
+	_downloader->requestedAmountIncrement(0, 0, -1);
 }
 
 class webFileLoaderPrivate {
@@ -1372,6 +1358,10 @@ WebLoadManager::WebLoadManager(QThread *thread) {
 #endif // OS_MAC_OLD
 }
 
+WebLoadManager::~WebLoadManager() {
+	clear();
+}
+
 void WebLoadManager::append(webFileLoader *loader, const QString &url) {
 	loader->_private = new webFileLoaderPrivate(loader, url);
 
@@ -1606,10 +1596,6 @@ void WebLoadManager::clear() {
 	_replies.clear();
 }
 
-WebLoadManager::~WebLoadManager() {
-	clear();
-}
-
 void WebLoadMainManager::progress(webFileLoader *loader, qint64 already, qint64 size) {
 	if (webLoadManager() && webLoadManager()->carries(loader)) {
 		loader->loadProgress(already, size);

Telegram/SourceFiles/storage/file_download.h

@@ -16,6 +16,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 #include <QtNetwork/QNetworkReply>
 
 class ApiWrap;
+class FileLoader;
 
 namespace Main {
 class Session;
@@ -35,48 +36,51 @@ constexpr auto kMaxWallPaperInMemory = kMaxFileInMemory;
 constexpr auto kMaxAnimationInMemory = kMaxFileInMemory; // 10 MB gif and mp4 animations held in memory while playing
 constexpr auto kMaxWallPaperDimension = 4096; // 4096x4096 is max area.
 
-class Downloader final {
+class Downloader final : public base::has_weak_ptr {
 public:
-	struct Queue {
-		Queue(int queriesLimit) : queriesLimit(queriesLimit) {
-		}
-		int queriesCount = 0;
-		int queriesLimit = 0;
-		FileLoader *start = nullptr;
-		FileLoader *end = nullptr;
-	};
-
 	explicit Downloader(not_null<ApiWrap*> api);
 	~Downloader();
 
-	ApiWrap &api() const {
+	[[nodiscard]] ApiWrap &api() const {
 		return *_api;
 	}
 
-	int currentPriority() const {
+	void enqueue(not_null<FileLoader*> loader);
-		return _priority;
+	void remove(not_null<FileLoader*> loader);
-	}
-	void clearPriorities();
 
-	base::Observable<void> &taskFinished() {
+	[[nodiscard]] base::Observable<void> &taskFinished() {
 		return _taskFinishedObservable;
 	}
 
+	// dcId == 0 is for web requests.
 	void requestedAmountIncrement(MTP::DcId dcId, int index, int amount);
-	int chooseDcIndexForRequest(MTP::DcId dcId) const;
+	[[nodiscard]] int chooseDcIndexForRequest(MTP::DcId dcId);
-
-	not_null<Queue*> queueForDc(MTP::DcId dcId);
-	not_null<Queue*> queueForWeb();
 
 private:
+	class Queue final {
+	public:
+		void enqueue(not_null<FileLoader*> loader);
+		void remove(not_null<FileLoader*> loader);
+		void resetGeneration();
+		[[nodiscard]] FileLoader *nextLoader() const;
+
+	private:
+		std::vector<not_null<FileLoader*>> _loaders;
+		std::vector<not_null<FileLoader*>> _previousGeneration;
+
+	};
+
+	void checkSendNext();
+
 	void killDownloadSessionsStart(MTP::DcId dcId);
 	void killDownloadSessionsStop(MTP::DcId dcId);
 	void killDownloadSessions();
 
-	not_null<ApiWrap*> _api;
+	void resetGeneration();
+
+	const not_null<ApiWrap*> _api;
 
 	base::Observable<void> _taskFinishedObservable;
-	int _priority = 1;
 
 	using RequestedInDc = std::array<int64, MTP::kDownloadSessionsCount>;
 	base::flat_map<MTP::DcId, RequestedInDc> _requestedBytesAmount;
@@ -84,8 +88,9 @@ private:
 	base::flat_map<MTP::DcId, crl::time> _killDownloadSessionTimes;
 	base::Timer _killDownloadSessionsTimer;
 
-	std::map<MTP::DcId, Queue> _queuesForDc;
+	base::flat_map<MTP::DcId, Queue> _mtprotoLoaders;
-	Queue _queueForWeb;
+	Queue _webLoaders;
+	bool _resettingGeneration = false;
 
 };
 
@@ -100,15 +105,13 @@ struct StorageImageSaved {
 
 };
 
-class mtpFileLoader;
-class webFileLoader;
-
 class FileLoader : public QObject {
 	Q_OBJECT
 
 public:
 	FileLoader(
 		const QString &toFile,
+		MTP::DcId dcId,
 		int32 size,
 		LocationType locationType,
 		LoadToCacheSetting toCache,
@@ -147,12 +150,6 @@ public:
 	void start();
 	void cancel();
 
-	bool loading() const {
-		return _inQueue;
-	}
-	bool started() const {
-		return _inQueue;
-	}
 	bool loadingLocal() const {
 		return (_localStatus == LocalStatus::Loading);
 	}
@@ -174,7 +171,7 @@ signals:
 	void failed(FileLoader *loader, bool started);
 
 protected:
-	using Queue = Storage::Downloader::Queue;
+	friend class Storage::Downloader;
 
 	enum class LocalStatus {
 		NotTried,
@@ -183,6 +180,10 @@ protected:
 		Loaded,
 	};
 
+	[[nodiscard]] MTP::DcId dcId() const {
+		return _dcId;
+	}
+
 	void readImage(const QSize &shrinkBox) const;
 
 	bool tryLoadLocal();
@@ -191,27 +192,21 @@ protected:
 	virtual std::optional<MediaKey> fileLocationKey() const = 0;
 	virtual void cancelRequests() = 0;
 
-	void startLoading();
-	void removeFromQueue();
 	void cancel(bool failed);
 
 	void notifyAboutProgress();
-	static void LoadNextFromQueue(not_null<Queue*> queue);
+	[[nodiscard]] virtual bool readyToRequest() const = 0;
-	virtual bool loadPart() = 0;
+	virtual void loadPart(int dcIndex) = 0;
 
 	bool writeResultPart(int offset, bytes::const_span buffer);
 	bool finalizeResult();
 	[[nodiscard]] QByteArray readLoadedPartBack(int offset, int size);
 
-	not_null<Storage::Downloader*> _downloader;
+	const MTP::DcId _dcId = 0;
-	FileLoader *_prev = nullptr;
+	const not_null<Storage::Downloader*> _downloader;
-	FileLoader *_next = nullptr;
-	int _priority = 0;
-	Queue *_queue = nullptr;
 
 	bool _autoLoading = false;
 	uint8 _cacheTag = 0;
-	bool _inQueue = false;
 	bool _finished = false;
 	bool _cancelled = false;
 	mutable LocalStatus _localStatus = LocalStatus::NotTried;
@@ -237,7 +232,7 @@ protected:
 
 class StorageImageLocation;
 class WebFileLocation;
-class mtpFileLoader : public FileLoader, public RPCSender {
+class mtpFileLoader final : public FileLoader, public RPCSender {
 public:
 	mtpFileLoader(
 		const StorageFileLocation &location,
@@ -277,6 +272,8 @@ public:
 	~mtpFileLoader();
 
 private:
+	friend class Downloader;
+
 	struct RequestData {
 		MTP::DcId dcId = 0;
 		int dcIndex = 0;
@@ -292,11 +289,12 @@ private:
 	std::optional<MediaKey> fileLocationKey() const override;
 	void cancelRequests() override;
 
-	MTP::DcId dcId() const;
+	[[nodiscard]] RequestData prepareRequest(int offset, int dcIndex) const;
-	RequestData prepareRequest(int offset) const;
+	void makeRequest(int offset, int dcIndex);
 	void makeRequest(int offset);
 
-	bool loadPart() override;
+	bool readyToRequest() const override;
+	void loadPart(int dcIndex) override;
 	void normalPartLoaded(const MTPupload_File &result, mtpRequestId requestId);
 	void webPartLoaded(const MTPupload_WebFile &result, mtpRequestId requestId);
 	void cdnPartLoaded(const MTPupload_CdnFile &result, mtpRequestId requestId);
@@ -334,7 +332,6 @@ private:
 		StorageFileLocation,
 		WebFileLocation,
 		GeoPointLocation> _location;
-
 	Data::FileOrigin _origin;
 
 	MTP::DcId _cdnDcId = 0;
@@ -349,7 +346,7 @@ private:
 
 class webFileLoaderPrivate;
 
-class webFileLoader : public FileLoader {
+class webFileLoader final : public FileLoader {
 public:
 	webFileLoader(
 		const QString &url,
@@ -370,11 +367,15 @@ public:
 
 	~webFileLoader();
 
-protected:
+private:
 	void cancelRequests() override;
 	Storage::Cache::Key cacheKey() const override;
 	std::optional<MediaKey> fileLocationKey() const override;
-	bool loadPart() override;
+	bool readyToRequest() const override;
+	void loadPart(int dcIndex) override;
+
+	void markAsSent();
+	void markAsNotSent();
 
 	QString _url;
 

Telegram/SourceFiles/storage/streamed_file_downloader.cpp

@@ -37,6 +37,7 @@ StreamedFileDownloader::StreamedFileDownloader(
 	uint8 cacheTag)
 : FileLoader(
 	toFile,
+	dcId,
 	size,
 	locationType,
 	toCache,
@@ -59,8 +60,6 @@ StreamedFileDownloader::StreamedFileDownloader(
 			savePart(std::move(part));
 		}
 	}, _lifetime);
-
-	_queue = _downloader->queueForDc(dcId);
 }
 
 StreamedFileDownloader::~StreamedFileDownloader() {
@@ -98,21 +97,25 @@ std::optional<MediaKey> StreamedFileDownloader::fileLocationKey() const {
 }
 
 void StreamedFileDownloader::cancelRequests() {
-	//_partsRequested == std::count(
-	//	begin(_partIsSaved),
-	//	begin(_partIsSaved) + _nextPartIndex,
-	//	false);
-	_queue->queriesCount -= _partsRequested;
 	_partsRequested = 0;
 	_nextPartIndex = 0;
 
 	_reader->cancelForDownloader(this);
 }
 
-bool StreamedFileDownloader::loadPart() {
+bool StreamedFileDownloader::readyToRequest() const {
 	if (_finished || _nextPartIndex >= _partsCount) {
 		return false;
 	}
+	_nextPartIndex = std::find(
+		begin(_partIsSaved) + _nextPartIndex,
+		end(_partIsSaved),
+		false
+	) - begin(_partIsSaved);
+	return (_nextPartIndex < _partsCount);
+}
+
+void StreamedFileDownloader::loadPart(int dcIndex) {
 	const auto index = std::find(
 		begin(_partIsSaved) + _nextPartIndex,
 		end(_partIsSaved),
@@ -120,15 +123,11 @@ bool StreamedFileDownloader::loadPart() {
 	) - begin(_partIsSaved);
 	if (index == _partsCount) {
 		_nextPartIndex = _partsCount;
-		return false;
+		return;
 	}
 	_nextPartIndex = index + 1;
 	_reader->loadForDownloader(this, index * kPartSize);
-
 	++_partsRequested;
-	++_queue->queriesCount;
-
-	return true;
 }
 
 void StreamedFileDownloader::savePart(const LoadedPart &part) {
@@ -150,7 +149,6 @@ void StreamedFileDownloader::savePart(const LoadedPart &part) {
 
 	if (index < _nextPartIndex) {
 		--_partsRequested;
-		--_queue->queriesCount;
 	}
 	if (!writeResultPart(offset, bytes::make_span(part.bytes))) {
 		return;

Telegram/SourceFiles/storage/streamed_file_downloader.h

@@ -49,7 +49,8 @@ private:
 	Cache::Key cacheKey() const override;
 	std::optional<MediaKey> fileLocationKey() const override;
 	void cancelRequests() override;
-	bool loadPart() override;
+	bool readyToRequest() const override;
+	void loadPart(int dcIndex) override;
 
 	void savePart(const Media::Streaming::LoadedPart &part);
 
@@ -60,7 +61,7 @@ private:
 	std::shared_ptr<Media::Streaming::Reader> _reader;
 
 	std::vector<bool> _partIsSaved; // vector<bool> :D
-	int _nextPartIndex = 0;
+	mutable int _nextPartIndex = 0;
 	int _partsCount = 0;
 	int _partsRequested = 0;
 	int _partsSaved = 0;