Skip to content

Parallel asset downloading isn't implemented correctly #556

Closed
@adazem009

Description

@adazem009

The algorithm is incorrect.

bool ProjectDownloader::downloadAssets(const std::vector<std::string> &assetIds)
{
m_cancelMutex.lock();
m_cancel = false;
m_cancelMutex.unlock();
auto count = assetIds.size();
unsigned int threadCount = std::thread::hardware_concurrency();
// Thread count: number of assets / 5, limited to maximum number of threads
threadCount = std::max(1u, std::min(threadCount, static_cast<unsigned int>(std::ceil(count / 5.0))));
m_assets.clear();
m_assets.reserve(count);
m_downloadedAssetCount = 0;
for (unsigned int i = 0; i < count; i++)
m_assets.push_back(std::string());
std::cout << "Downloading " << count << " asset(s)";
if (threadCount > 1)
std::cout << " using " << threadCount << " threads";
std::cout << std::endl;
// Create downloaders
std::vector<std::shared_ptr<IDownloader>> downloaders;
for (unsigned int i = 0; i < threadCount; i++)
downloaders.push_back(m_downloaderFactory->create());
// Download assets
auto f = [this, &downloaders, &assetIds, count, threadCount](unsigned int thread) {
auto downloader = downloaders[thread];
unsigned int n = std::ceil(count / static_cast<double>(threadCount));
for (unsigned int i = 0; i < n; i++) {
unsigned int index = thread * n + i;
if (index < count) {
m_cancelMutex.lock();
if (m_cancel)
return;
m_cancelMutex.unlock();
bool ret = downloader->download(ASSET_PREFIX + assetIds[index] + ASSET_SUFFIX);
if (!ret) {
std::cerr << "Failed to download asset: " << assetIds[index] << std::endl;
m_cancelMutex.lock();
m_cancel = true;
m_cancelMutex.unlock();
return;
}
m_assetsMutex.lock();
m_assets[index] = downloader->text();
m_downloadedAssetCount++;
std::cout << "Downloaded assets: " << m_downloadedAssetCount << " of " << count << std::endl;
m_downloadProgressChanged(m_downloadedAssetCount, count);
m_assetsMutex.unlock();
}
}
};
std::vector<std::thread> threads;
for (unsigned int i = 0; i < threadCount; i++)
threads.emplace_back(std::thread(f, i));
for (unsigned int i = 0; i < threadCount; i++)
threads[i].join();
CHECK_CANCEL();
return true;
}

Metadata

Metadata

Assignees

No one assigned

    Labels

    P1Priority: HighbugSomething isn't working

    Type

    No type

    Projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions