diff --git a/src/auto_archiver/archivers/archiver.py b/src/auto_archiver/archivers/archiver.py index 9156570..e10beda 100644 --- a/src/auto_archiver/archivers/archiver.py +++ b/src/auto_archiver/archivers/archiver.py @@ -3,6 +3,8 @@ from abc import abstractmethod from dataclasses import dataclass import os import mimetypes, requests +from loguru import logger +from retrying import retry from ..core import Metadata, Step, ArchivingContext @@ -41,6 +43,7 @@ class Archiver(Step): return mime.split("/")[0] return "" + @retry(wait_random_min=500, wait_random_max=3500, stop_max_attempt_number=5) def download_from_url(self, url: str, to_filename: str = None) -> str: """ downloads a URL to provided filename, or inferred from URL, returns local filename @@ -50,6 +53,7 @@ class Archiver(Step): if len(to_filename) > 64: to_filename = to_filename[-64:] to_filename = os.path.join(ArchivingContext.get_tmp_dir(), to_filename) + logger.debug(f"downloading {url[0:50]=} {to_filename=}") headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36' }