mirror
/
OpenDroneMap-WebODM
kopia lustrzana https://github.com/OpenDroneMap/WebODM
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

feat: parallel downloads back-off

pull/838/head
Piero Toffanin 2020-03-23 20:34:07 -04:00
rodzic 217990288d
commit 927b7795f3
2 zmienionych plików z 5 dodań i 5 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

6
app/models/task.py
Wyświetl plik

@ -629,7 +629,7 @@ class Task(models.Model):
os.makedirs(assets_dir)
# Download and try to extract results up to 4 times
# (~95% of the times, on large downloads, the archive could be corrupted)
# (~5% of the times, on large downloads, the archive could be corrupted)
retry_num = 0
extracted = False
last_update = 0
@ -649,7 +649,7 @@ class Task(models.Model):
logger.info("Downloading all.zip for {}".format(self))
# Download all assets
zip_path = self.processing_node.download_task_assets(self.uuid, assets_dir, progress_callback=callback)
zip_path = self.processing_node.download_task_assets(self.uuid, assets_dir, progress_callback=callback, parallel_downloads=max(1, int(16 / (2 ** retry_num))))
# Rename to all.zip
all_zip_path = self.assets_path("all.zip")
@ -661,7 +661,7 @@ class Task(models.Model):
self.extract_assets_and_complete()
extracted = True
except zipfile.BadZipFile:
if retry_num < 4:
if retry_num < 5:
logger.warning("{} seems corrupted. Retrying...".format(all_zip_path))
retry_num += 1
os.remove(all_zip_path)

4
nodeodm/models.py
Wyświetl plik

@ -160,13 +160,13 @@ class ProcessingNode(models.Model):
task = api_client.get_task(uuid)
return task.remove()
def download_task_assets(self, uuid, destination, progress_callback):
def download_task_assets(self, uuid, destination, progress_callback, parallel_downloads=16):
"""
Downloads a task asset
"""
api_client = self.api_client()
task = api_client.get_task(uuid)
return task.download_zip(destination, progress_callback)
return task.download_zip(destination, progress_callback, parallel_downloads=parallel_downloads)
def restart_task(self, uuid, options = None):
"""