From 4a738721f1b40aac721088b7c2300fafe8b57b1b Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Fri, 25 Sep 2020 09:58:49 +1000 Subject: [PATCH] idf_tools: Add option to replace all GitHub tools download URLs with dl.espressif.com Via new IDF_GITHUB_ASSETS environment variable. --- docs/en/api-guides/tools/idf-tools.rst | 14 +++++++++++ docs/en/get-started/index.rst | 27 ++++++++++++++++++++ tools/idf_tools.py | 35 ++++++++++++++++++++++++-- 3 files changed, 74 insertions(+), 2 deletions(-) diff --git a/docs/en/api-guides/tools/idf-tools.rst b/docs/en/api-guides/tools/idf-tools.rst index 6ea511a3ff..7c2cfe2044 100644 --- a/docs/en/api-guides/tools/idf-tools.rst +++ b/docs/en/api-guides/tools/idf-tools.rst @@ -40,6 +40,20 @@ Inside ``IDF_TOOLS_PATH``, the scripts performing tools installation create the - ``dist`` — where the archives of the tools are downloaded. - ``tools`` — where the tools are extracted. The tools are extracted into subdirectories: ``tools/TOOL_NAME/VERSION/``. This arrangement allows different versions of tools to be installed side by side. +GitHub Assets Mirror +-------------------- + +Most of the tools downloaded by the tools installer are GitHub Release Assets, which are files attached to a software release on GitHub. + +If GitHub downloads are inaccessible or slow to access, it's possible to configure a GitHub assets mirror. + +To use Espressif's download server, set the environment variable ``IDF_GITHUB_ASSETS`` to ``dl.espressif.com/github_assets``. When the install process is downloading a tool from ``github.com``, the URL will be rewritten to use this server instead. + +Any mirror server can be used provided the URL matches the ``github.com`` download URL format: the install process will replace ``https://github.com`` with ``https://${IDF_GITHUB_ASSETS}`` for any GitHub asset URL that it downloads. + +.. note:: The Espressif download server doesn't currently mirror everything from GitHub, it only mirrors files attached as Assets to some releases as well as source archives for some releases. + + ``idf_tools.py`` script ----------------------- diff --git a/docs/en/get-started/index.rst b/docs/en/get-started/index.rst index 2ec9911234..e80da35e40 100644 --- a/docs/en/get-started/index.rst +++ b/docs/en/get-started/index.rst @@ -227,6 +227,33 @@ Linux and macOS cd ~/esp/esp-idf ./install.sh +Alternative File Downloads +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The tools installer downloads a number of files attached to GitHub Releases. If accessing GitHub is slow then it is possible to set an environment variable to prefer Espressif's download server for GitHub asset downloads. + +.. note:: This setting only controls individual tools downloaded from GitHub releases, it doesn't change the URLs used to access any Git repositories. + +Windows +------- + +To prefer the Espressif download server when running the ESP-IDF Tools Installer or installing tools from the command line, open the System control panel, then click on Advanced Settings. Add a new Environment Variable (of type either User or System) with the name ``IDF_GITHUB_ASSETS`` and value ``dl.espressif.com/github_assets``. Click OK once done. + +If the command line window or ESP-IDF Tools Installer window was already open before you added the new environment variable, you will need to close and reopen it. + +While this environment variable is still set, the ESP-IDF Tools Installer and the command line installer will prefer the Espressif download server. + +Linux and macOS +--------------- + +To prefer the Espressif download server when installing tools, use the following sequence of commands when running ``install.sh``: + +.. code-block:: bash + + cd ~/esp/esp-idf + export IDF_GITHUB_ASSETS="dl.espressif.com/github_assets" + ./install.sh + Customizing the tools installation path ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tools/idf_tools.py b/tools/idf_tools.py index 7dbe56a4d1..f060ca4ada 100755 --- a/tools/idf_tools.py +++ b/tools/idf_tools.py @@ -1163,6 +1163,11 @@ def action_export(args): raise SystemExit(1) +def apply_url_mirrors(args, tool_download_obj): + apply_mirror_prefix_map(args, tool_download_obj) + apply_github_assets_option(tool_download_obj) + + def apply_mirror_prefix_map(args, tool_download_obj): """Rewrite URL for given tool_obj, given tool_version, and current platform, if --mirror-prefix-map flag or IDF_MIRROR_PREFIX_MAP environment variable is given. @@ -1190,6 +1195,32 @@ def apply_mirror_prefix_map(args, tool_download_obj): break +def apply_github_assets_option(tool_download_obj): + """ Rewrite URL for given tool_obj if the download URL is an https://github.com/ URL and the variable + IDF_GITHUB_ASSETS is set. The github.com part of the URL will be replaced. + """ + try: + github_assets = os.environ["IDF_GITHUB_ASSETS"].strip() + except KeyError: + return # no IDF_GITHUB_ASSETS + if not github_assets: # variable exists but is empty + return + + # check no URL qualifier in the mirror URL + if '://' in github_assets: + fatal("IDF_GITHUB_ASSETS shouldn't include any URL qualifier, https:// is assumed") + raise SystemExit(1) + + # Strip any trailing / from the mirror URL + github_assets = github_assets.rstrip('/') + + old_url = tool_download_obj.url + new_url = re.sub(r'^https://github.com/', 'https://{}/'.format(github_assets), old_url) + if new_url != old_url: + info('Using GitHub assets mirror for URL: {} => {}'.format(old_url, new_url)) + tool_download_obj.url = new_url + + def action_download(args): tools_info = load_tools_info() tools_spec = args.tools @@ -1232,7 +1263,7 @@ def action_download(args): tool_spec = '{}@{}'.format(tool_name, tool_version) info('Downloading {}'.format(tool_spec)) - apply_mirror_prefix_map(args, tool_obj.versions[tool_version].get_download_for_platform(platform)) + apply_url_mirrors(args, tool_obj.versions[tool_version].get_download_for_platform(platform)) tool_obj.download(tool_version) @@ -1273,7 +1304,7 @@ def action_install(args): continue info('Installing {}'.format(tool_spec)) - apply_mirror_prefix_map(args, tool_obj.versions[tool_version].get_download_for_platform(PYTHON_PLATFORM)) + apply_url_mirrors(args, tool_obj.versions[tool_version].get_download_for_platform(PYTHON_PLATFORM)) tool_obj.download(tool_version) tool_obj.install(tool_version)