diff --git a/app/models/task.py b/app/models/task.py index 8d0d04cf..53c80c10 100644 --- a/app/models/task.py +++ b/app/models/task.py @@ -629,7 +629,7 @@ class Task(models.Model): os.makedirs(assets_dir) # Download and try to extract results up to 4 times - # (~95% of the times, on large downloads, the archive could be corrupted) + # (~5% of the times, on large downloads, the archive could be corrupted) retry_num = 0 extracted = False last_update = 0 @@ -649,7 +649,7 @@ class Task(models.Model): logger.info("Downloading all.zip for {}".format(self)) # Download all assets - zip_path = self.processing_node.download_task_assets(self.uuid, assets_dir, progress_callback=callback) + zip_path = self.processing_node.download_task_assets(self.uuid, assets_dir, progress_callback=callback, parallel_downloads=max(1, int(16 / (2 ** retry_num)))) # Rename to all.zip all_zip_path = self.assets_path("all.zip") @@ -661,7 +661,7 @@ class Task(models.Model): self.extract_assets_and_complete() extracted = True except zipfile.BadZipFile: - if retry_num < 4: + if retry_num < 5: logger.warning("{} seems corrupted. Retrying...".format(all_zip_path)) retry_num += 1 os.remove(all_zip_path) diff --git a/nodeodm/models.py b/nodeodm/models.py index cd7a47ad..b549809a 100644 --- a/nodeodm/models.py +++ b/nodeodm/models.py @@ -160,13 +160,13 @@ class ProcessingNode(models.Model): task = api_client.get_task(uuid) return task.remove() - def download_task_assets(self, uuid, destination, progress_callback): + def download_task_assets(self, uuid, destination, progress_callback, parallel_downloads=16): """ Downloads a task asset """ api_client = self.api_client() task = api_client.get_task(uuid) - return task.download_zip(destination, progress_callback) + return task.download_zip(destination, progress_callback, parallel_downloads=parallel_downloads) def restart_task(self, uuid, options = None): """