diff --git a/tools/ci/mypy_ignore_list.txt b/tools/ci/mypy_ignore_list.txt index 69c6834f0e..2b0fb4ea29 100644 --- a/tools/ci/mypy_ignore_list.txt +++ b/tools/ci/mypy_ignore_list.txt @@ -167,7 +167,6 @@ tools/ci/ci_fetch_submodule.py tools/ci/deploy_docs.py tools/ci/envsubst.py tools/ci/normalize_clangtidy_path.py -tools/ci/python_packages/gitlab_api.py tools/ci/python_packages/idf_http_server_test/adder.py tools/ci/python_packages/idf_http_server_test/client.py tools/ci/python_packages/idf_http_server_test/test.py diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py index c6d6fd7a76..f380bc1c86 100644 --- a/tools/ci/python_packages/gitlab_api.py +++ b/tools/ci/python_packages/gitlab_api.py @@ -3,27 +3,63 @@ import os import re import tarfile import tempfile +import time import zipfile from functools import wraps +from typing import Any, Callable, Dict, List, Optional import gitlab +TR = Callable[..., Any] + + +def retry(func: TR) -> TR: + """ + This wrapper will only catch several exception types associated with + "network issues" and retry the whole function. + """ + @wraps(func) + def wrapper(self: 'Gitlab', *args: Any, **kwargs: Any) -> Any: + retried = 0 + while True: + try: + res = func(self, *args, **kwargs) + except (IOError, EOFError, gitlab.exceptions.GitlabError) as e: + if isinstance(e, gitlab.exceptions.GitlabError) and e.response_code != 500: + # Only retry on error 500 + raise e + retried += 1 + if retried > self.DOWNLOAD_ERROR_MAX_RETRIES: + raise e # get out of the loop + else: + print('Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'), retried)) + time.sleep(2 ** retried) # wait a bit more after each retry + continue + else: + break + return res + return wrapper + class Gitlab(object): JOB_NAME_PATTERN = re.compile(r'(\w+)(\s+(\d+)/(\d+))?') DOWNLOAD_ERROR_MAX_RETRIES = 3 - def __init__(self, project_id=None): + def __init__(self, project_id: Optional[int] = None): config_data_from_env = os.getenv('PYTHON_GITLAB_CONFIG') if config_data_from_env: # prefer to load config from env variable with tempfile.NamedTemporaryFile('w', delete=False) as temp_file: temp_file.write(config_data_from_env) - config_files = [temp_file.name] + config_files = [temp_file.name] # type: Optional[List[str]] else: # otherwise try to use config file at local filesystem config_files = None + self._init_gitlab_inst(project_id, config_files) + + @retry + def _init_gitlab_inst(self, project_id: Optional[int], config_files: Optional[List[str]]) -> None: gitlab_id = os.getenv('LOCAL_GITLAB_HTTPS_HOST') # if None, will use the default gitlab server self.gitlab_inst = gitlab.Gitlab.from_config(gitlab_id=gitlab_id, config_files=config_files) self.gitlab_inst.auth() @@ -32,7 +68,8 @@ class Gitlab(object): else: self.project = None - def get_project_id(self, name, namespace=None): + @retry + def get_project_id(self, name: str, namespace: Optional[str] = None) -> int: """ search project ID by name @@ -56,9 +93,10 @@ class Gitlab(object): if not res: raise ValueError("Can't find project") - return res[0] + return int(res[0]) - def download_artifacts(self, job_id, destination): + @retry + def download_artifacts(self, job_id: int, destination: str) -> None: """ download full job artifacts and extract to destination. @@ -73,33 +111,8 @@ class Gitlab(object): with zipfile.ZipFile(temp_file.name, 'r') as archive_file: archive_file.extractall(destination) - def retry_download(func): - """ - This wrapper will only catch IOError and retry the whole function. - - So only use it with download functions, read() inside and atomic - functions - """ - @wraps(func) - def wrapper(self, *args, **kwargs): - retried = 0 - while True: - try: - res = func(self, *args, **kwargs) - except (IOError, EOFError) as e: - retried += 1 - if retried > self.DOWNLOAD_ERROR_MAX_RETRIES: - raise e # get out of the loop - else: - print('Retried for the {} time'.format(retried)) - continue - else: - break - return res - return wrapper - - @retry_download # type: ignore - def download_artifact(self, job_id, artifact_path, destination=None): + @retry + def download_artifact(self, job_id: int, artifact_path: str, destination: Optional[str] = None) -> List[bytes]: """ download specific path of job artifacts and extract to destination. @@ -114,7 +127,7 @@ class Gitlab(object): for a_path in artifact_path: try: - data = job.artifact(a_path) + data = job.artifact(a_path) # type: bytes except gitlab.GitlabGetError as e: print("Failed to download '{}' from job {}".format(a_path, job_id)) raise e @@ -131,7 +144,8 @@ class Gitlab(object): return raw_data_list - def find_job_id(self, job_name, pipeline_id=None, job_status='success'): + @retry + def find_job_id(self, job_name: str, pipeline_id: Optional[str] = None, job_status: str = 'success') -> List[Dict]: """ Get Job ID from job name of specific pipeline @@ -153,8 +167,8 @@ class Gitlab(object): job_id_list.append({'id': job.id, 'parallel_num': match.group(3)}) return job_id_list - @retry_download # type: ignore - def download_archive(self, ref, destination, project_id=None): + @retry + def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None) -> str: """ Download archive of certain commit of a repository and extract to destination path @@ -184,7 +198,7 @@ class Gitlab(object): return os.path.join(os.path.realpath(destination), root_name) -if __name__ == '__main__': +def main() -> None: parser = argparse.ArgumentParser() parser.add_argument('action') parser.add_argument('project_id', type=int) @@ -210,3 +224,7 @@ if __name__ == '__main__': elif args.action == 'get_project_id': ret = gitlab_inst.get_project_id(args.project_name) print('project id: {}'.format(ret)) + + +if __name__ == '__main__': + main()