manual download archiver improvements

src/auto_archiver/archivers/archiver.py

@@ -3,6 +3,8 @@ from abc import abstractmethod
 from dataclasses import dataclass
 import os
 import mimetypes, requests
+from loguru import logger
+from retrying import retry
 
 from ..core import Metadata, Step, ArchivingContext
 
@@ -41,6 +43,7 @@ class Archiver(Step):
             return mime.split("/")[0]
         return ""
 
+    @retry(wait_random_min=500, wait_random_max=3500, stop_max_attempt_number=5)
     def download_from_url(self, url: str, to_filename: str = None) -> str:
         """
         downloads a URL to provided filename, or inferred from URL, returns local filename
@@ -50,6 +53,7 @@ class Archiver(Step):
             if len(to_filename) > 64:
                 to_filename = to_filename[-64:]
         to_filename = os.path.join(ArchivingContext.get_tmp_dir(), to_filename)
+        logger.debug(f"downloading {url[0:50]=} {to_filename=}")
         headers = {
             'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
         }