From 4a738721f1b40aac721088b7c2300fafe8b57b1b Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Fri, 25 Sep 2020 09:58:49 +1000 Subject: [PATCH 1/5] idf_tools: Add option to replace all GitHub tools download URLs with dl.espressif.com Via new IDF_GITHUB_ASSETS environment variable. --- docs/en/api-guides/tools/idf-tools.rst | 14 +++++++++++ docs/en/get-started/index.rst | 27 ++++++++++++++++++++ tools/idf_tools.py | 35 ++++++++++++++++++++++++-- 3 files changed, 74 insertions(+), 2 deletions(-) diff --git a/docs/en/api-guides/tools/idf-tools.rst b/docs/en/api-guides/tools/idf-tools.rst index 6ea511a3ff..7c2cfe2044 100644 --- a/docs/en/api-guides/tools/idf-tools.rst +++ b/docs/en/api-guides/tools/idf-tools.rst @@ -40,6 +40,20 @@ Inside ``IDF_TOOLS_PATH``, the scripts performing tools installation create the - ``dist`` — where the archives of the tools are downloaded. - ``tools`` — where the tools are extracted. The tools are extracted into subdirectories: ``tools/TOOL_NAME/VERSION/``. This arrangement allows different versions of tools to be installed side by side. +GitHub Assets Mirror +-------------------- + +Most of the tools downloaded by the tools installer are GitHub Release Assets, which are files attached to a software release on GitHub. + +If GitHub downloads are inaccessible or slow to access, it's possible to configure a GitHub assets mirror. + +To use Espressif's download server, set the environment variable ``IDF_GITHUB_ASSETS`` to ``dl.espressif.com/github_assets``. When the install process is downloading a tool from ``github.com``, the URL will be rewritten to use this server instead. + +Any mirror server can be used provided the URL matches the ``github.com`` download URL format: the install process will replace ``https://github.com`` with ``https://${IDF_GITHUB_ASSETS}`` for any GitHub asset URL that it downloads. + +.. note:: The Espressif download server doesn't currently mirror everything from GitHub, it only mirrors files attached as Assets to some releases as well as source archives for some releases. + + ``idf_tools.py`` script ----------------------- diff --git a/docs/en/get-started/index.rst b/docs/en/get-started/index.rst index 2ec9911234..e80da35e40 100644 --- a/docs/en/get-started/index.rst +++ b/docs/en/get-started/index.rst @@ -227,6 +227,33 @@ Linux and macOS cd ~/esp/esp-idf ./install.sh +Alternative File Downloads +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The tools installer downloads a number of files attached to GitHub Releases. If accessing GitHub is slow then it is possible to set an environment variable to prefer Espressif's download server for GitHub asset downloads. + +.. note:: This setting only controls individual tools downloaded from GitHub releases, it doesn't change the URLs used to access any Git repositories. + +Windows +------- + +To prefer the Espressif download server when running the ESP-IDF Tools Installer or installing tools from the command line, open the System control panel, then click on Advanced Settings. Add a new Environment Variable (of type either User or System) with the name ``IDF_GITHUB_ASSETS`` and value ``dl.espressif.com/github_assets``. Click OK once done. + +If the command line window or ESP-IDF Tools Installer window was already open before you added the new environment variable, you will need to close and reopen it. + +While this environment variable is still set, the ESP-IDF Tools Installer and the command line installer will prefer the Espressif download server. + +Linux and macOS +--------------- + +To prefer the Espressif download server when installing tools, use the following sequence of commands when running ``install.sh``: + +.. code-block:: bash + + cd ~/esp/esp-idf + export IDF_GITHUB_ASSETS="dl.espressif.com/github_assets" + ./install.sh + Customizing the tools installation path ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tools/idf_tools.py b/tools/idf_tools.py index 7dbe56a4d1..f060ca4ada 100755 --- a/tools/idf_tools.py +++ b/tools/idf_tools.py @@ -1163,6 +1163,11 @@ def action_export(args): raise SystemExit(1) +def apply_url_mirrors(args, tool_download_obj): + apply_mirror_prefix_map(args, tool_download_obj) + apply_github_assets_option(tool_download_obj) + + def apply_mirror_prefix_map(args, tool_download_obj): """Rewrite URL for given tool_obj, given tool_version, and current platform, if --mirror-prefix-map flag or IDF_MIRROR_PREFIX_MAP environment variable is given. @@ -1190,6 +1195,32 @@ def apply_mirror_prefix_map(args, tool_download_obj): break +def apply_github_assets_option(tool_download_obj): + """ Rewrite URL for given tool_obj if the download URL is an https://github.com/ URL and the variable + IDF_GITHUB_ASSETS is set. The github.com part of the URL will be replaced. + """ + try: + github_assets = os.environ["IDF_GITHUB_ASSETS"].strip() + except KeyError: + return # no IDF_GITHUB_ASSETS + if not github_assets: # variable exists but is empty + return + + # check no URL qualifier in the mirror URL + if '://' in github_assets: + fatal("IDF_GITHUB_ASSETS shouldn't include any URL qualifier, https:// is assumed") + raise SystemExit(1) + + # Strip any trailing / from the mirror URL + github_assets = github_assets.rstrip('/') + + old_url = tool_download_obj.url + new_url = re.sub(r'^https://github.com/', 'https://{}/'.format(github_assets), old_url) + if new_url != old_url: + info('Using GitHub assets mirror for URL: {} => {}'.format(old_url, new_url)) + tool_download_obj.url = new_url + + def action_download(args): tools_info = load_tools_info() tools_spec = args.tools @@ -1232,7 +1263,7 @@ def action_download(args): tool_spec = '{}@{}'.format(tool_name, tool_version) info('Downloading {}'.format(tool_spec)) - apply_mirror_prefix_map(args, tool_obj.versions[tool_version].get_download_for_platform(platform)) + apply_url_mirrors(args, tool_obj.versions[tool_version].get_download_for_platform(platform)) tool_obj.download(tool_version) @@ -1273,7 +1304,7 @@ def action_install(args): continue info('Installing {}'.format(tool_spec)) - apply_mirror_prefix_map(args, tool_obj.versions[tool_version].get_download_for_platform(PYTHON_PLATFORM)) + apply_url_mirrors(args, tool_obj.versions[tool_version].get_download_for_platform(PYTHON_PLATFORM)) tool_obj.download(tool_version) tool_obj.install(tool_version) From 291af6e7660aafc83fbc058229bf7dafc9c94bf7 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Fri, 25 Sep 2020 19:19:17 +1000 Subject: [PATCH 2/5] tools: Use GitHub download URLs for all files that can be downloaded from GitHub --- docs/en/get-started/index.rst | 2 ++ docs/en/get-started/windows-setup.rst | 2 ++ tools/tools.json | 24 ++++++++++++------------ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/docs/en/get-started/index.rst b/docs/en/get-started/index.rst index e80da35e40..397a156d02 100644 --- a/docs/en/get-started/index.rst +++ b/docs/en/get-started/index.rst @@ -243,6 +243,8 @@ If the command line window or ESP-IDF Tools Installer window was already open be While this environment variable is still set, the ESP-IDF Tools Installer and the command line installer will prefer the Espressif download server. +.. Once the ESP-IDF Tools Installer binary is updated to include the checkbox, the above can be rewritten to refer to it + Linux and macOS --------------- diff --git a/docs/en/get-started/windows-setup.rst b/docs/en/get-started/windows-setup.rst index 4e226325c8..70d6d3b06d 100644 --- a/docs/en/get-started/windows-setup.rst +++ b/docs/en/get-started/windows-setup.rst @@ -36,6 +36,8 @@ The easiest way to install ESP-IDF's prerequisites is to download the ESP-IDF To https://dl.espressif.com/dl/esp-idf-tools-setup-2.3.exe +.. IMPORTANT: Next time this link is updated, please go to get-started/index.rst and rewrite the section under "Alternative File Downloads ... Windows". Then delete this comment. + The installer includes the cross-compilers, OpenOCD, CMake_ and Ninja_ build tool. The installer can also download and run installers for Python_ 3.7 and `Git For Windows`_ if they are not already installed on the computer. The installer also offers to download one of the ESP-IDF release versions. Please choose a directory for downloading ESP-IDF. The recommended directory is ``%userprofile%\esp`` where ``%userprofile%`` is your home directory. If you do not have it yet, please run the following command to create a new one: diff --git a/tools/tools.json b/tools/tools.json index 64063885e9..3fa458330e 100644 --- a/tools/tools.json +++ b/tools/tools.json @@ -24,34 +24,34 @@ "linux-amd64": { "sha256": "674080a12f9c5ebe5a3a5ce51c6deaeffe6dfb06d6416233df86f25b574e9279", "size": 85731226, - "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-gcc8_4_0-esp-2020r3-linux-amd64.tar.gz" + "url": "https://github.com/espressif/crosstool-NG/releases/download/esp-2020r3/xtensa-esp32-elf-gcc8_4_0-esp-2020r3-linux-amd64.tar.gz" }, "linux-armel": { "sha256": "6771e011dffa2438ef84ff3474538b4a69df8f9d4cfae3b3707ca31c782ed7db", "size": 83888892, - "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-gcc8_4_0-esp-2020r3-linux-armel.tar.gz" + "url": "https://github.com/espressif/crosstool-NG/releases/download/esp-2020r3/xtensa-esp32-elf-gcc8_4_0-esp-2020r3-linux-armel.tar.gz" }, "linux-i686": { "sha256": "076b7e05304e26aa6ec105c9e0dc74addca079bc2cae6e42ee7575c5ded29877", "size": 87715092, - "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-gcc8_4_0-esp-2020r3-linux-i686.tar.gz" + "url": "https://github.com/espressif/crosstool-NG/releases/download/esp-2020r3/xtensa-esp32-elf-gcc8_4_0-esp-2020r3-linux-i686.tar.gz" }, "macos": { "sha256": "6845f786303b26c4a55ede57487ba65bd25737232fe6104be03f25bb62f4631c", "size": 92424226, - "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-gcc8_4_0-esp-2020r3-macos.tar.gz" + "url": "https://github.com/espressif/crosstool-NG/releases/download/esp-2020r3/xtensa-esp32-elf-gcc8_4_0-esp-2020r3-macos.tar.gz" }, "name": "esp-2020r3-8.4.0", "status": "recommended", "win32": { "sha256": "81cecd5493a3fcf2118977f3fd60bd0a13a4aeac8fe6760d912f96d2c34fab66", "size": 104226379, - "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-gcc8_4_0-esp-2020r3-win32.zip" + "url": "https://github.com/espressif/crosstool-NG/releases/download/esp-2020r3/xtensa-esp32-elf-gcc8_4_0-esp-2020r3-win32.zip" }, "win64": { "sha256": "58419852fefb7111fdec056ac2fd7c4bd09c1f4c17610a761a97b788413527cf", "size": 106855139, - "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-gcc8_4_0-esp-2020r3-win64.zip" + "url": "https://github.com/espressif/crosstool-NG/releases/download/esp-2020r3/xtensa-esp32-elf-gcc8_4_0-esp-2020r3-win64.zip" } } ] @@ -80,34 +80,34 @@ "linux-amd64": { "sha256": "40fafa47045167feda0cd07827db5207ebfeb4a3b6b24475957a921bc92805ed", "size": 86069526, - "url": "https://dl.espressif.com/dl/xtensa-esp32s2-elf-gcc8_4_0-esp-2020r3-linux-amd64.tar.gz" + "url": "https://github.com/espressif/crosstool-NG/releases/download/esp-2020r3/xtensa-esp32s2-elf-gcc8_4_0-esp-2020r3-linux-amd64.tar.gz" }, "linux-armel": { "sha256": "6c1efec4c7829202279388ccb388e8a17a34464bc351d677c4f04d95ea4b4ce0", "size": 84254468, - "url": "https://dl.espressif.com/dl/xtensa-esp32s2-elf-gcc8_4_0-esp-2020r3-linux-armel.tar.gz" + "url": "https://github.com/espressif/crosstool-NG/releases/download/esp-2020r3/xtensa-esp32s2-elf-gcc8_4_0-esp-2020r3-linux-armel.tar.gz" }, "linux-i686": { "sha256": "bd3a91166206a1a7ff7c572e15389e1938c3cdce588032a5e915be677a945638", "size": 88053499, - "url": "https://dl.espressif.com/dl/xtensa-esp32s2-elf-gcc8_4_0-esp-2020r3-linux-i686.tar.gz" + "url": "https://github.com/espressif/crosstool-NG/releases/download/esp-2020r3/xtensa-esp32s2-elf-gcc8_4_0-esp-2020r3-linux-i686.tar.gz" }, "macos": { "sha256": "fe19b0c873879d8d89ec040f4db04a3ab27d769d3fd5f55fe59a28b6b111d09c", "size": 92817351, - "url": "https://dl.espressif.com/dl/xtensa-esp32s2-elf-gcc8_4_0-esp-2020r3-macos.tar.gz" + "url": "https://github.com/espressif/crosstool-NG/releases/download/esp-2020r3/xtensa-esp32s2-elf-gcc8_4_0-esp-2020r3-macos.tar.gz" }, "name": "esp-2020r3-8.4.0", "status": "recommended", "win32": { "sha256": "d078d614ae864ae4a37fcb5b83323af0a5cfdbd8243607664becdd0f977a1e33", "size": 104659541, - "url": "https://dl.espressif.com/dl/xtensa-esp32s2-elf-gcc8_4_0-esp-2020r3-win32.zip" + "url": "https://github.com/espressif/crosstool-NG/releases/download/esp-2020r3/xtensa-esp32s2-elf-gcc8_4_0-esp-2020r3-win32.zip" }, "win64": { "sha256": "6ea78b72ec52330b69af91dbe6c77c22bdc827817f044aa30306270453032bb4", "size": 107228353, - "url": "https://dl.espressif.com/dl/xtensa-esp32s2-elf-gcc8_4_0-esp-2020r3-win64.zip" + "url": "https://github.com/espressif/crosstool-NG/releases/download/esp-2020r3/xtensa-esp32s2-elf-gcc8_4_0-esp-2020r3-win64.zip" } } ] From 0e6d32cbfdc7b6e85d05f906cd0f2e1d859c4a43 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Fri, 25 Sep 2020 13:40:23 +0200 Subject: [PATCH 3/5] tools: windows installer: add support for IDF_GITHUB_ASSETS Adds a checkbox to download tools from dl.espressif.com mirror. --- tools/windows/tool_setup/idf_tool_setup.iss | 1 + tools/windows/tool_setup/main.iss.inc | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/tools/windows/tool_setup/idf_tool_setup.iss b/tools/windows/tool_setup/idf_tool_setup.iss index 83eb25f2e2..614203362b 100644 --- a/tools/windows/tool_setup/idf_tool_setup.iss +++ b/tools/windows/tool_setup/idf_tool_setup.iss @@ -82,6 +82,7 @@ Name: createdsk; Description: "Create Desktop shortcut for the ESP-IDF Tools Com ; WD registration checkbox is identified by 'Windows Defender' substring anywhere in its caption, not by the position index in WizardForm.TasksList.Items ; Please, keep this in mind when making changes to the item's description - WD checkbox is to be disabled on systems without the Windows Defender installed Name: wdexcl; Description: "Register the ESP-IDF Tools executables as Windows Defender exclusions (improves compilation speed, requires elevation)"; +Name: idf_tools_use_mirror; Description: "Use Espressif download server instead of downloading tool packages from Github"; Flags: unchecked; [Run] Filename: "{app}\dist\{#PythonInstallerName}"; Parameters: "/passive PrependPath=1 InstallLauncherAllUsers=0 Include_dev=0 Include_tcltk=0 Include_launcher=0 Include_test=0 Include_doc=0"; Description: "Installing Python"; Check: PythonInstallRequired diff --git a/tools/windows/tool_setup/main.iss.inc b/tools/windows/tool_setup/main.iss.inc index d54d6d9828..f6b0a50b47 100644 --- a/tools/windows/tool_setup/main.iss.inc +++ b/tools/windows/tool_setup/main.iss.inc @@ -117,6 +117,11 @@ begin if not IDFUseExisting then IDFDownload(); + if WizardIsTaskSelected('idf_tools_use_mirror') then + begin + SetEnvironmentVariable('IDF_GITHUB_ASSETS', 'dl.espressif.com/github_assets') + end; + IDFToolsSetup(); From 3efc9f3a04b35cc63c19f9c1c350bdca9572ee0e Mon Sep 17 00:00:00 2001 From: Anton Maklakov Date: Wed, 25 Nov 2020 17:37:25 +0700 Subject: [PATCH 4/5] tools: windows installer: Modify IDF's archive mirror link --- tools/windows/tool_setup/idf_setup.iss.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/windows/tool_setup/idf_setup.iss.inc b/tools/windows/tool_setup/idf_setup.iss.inc index 303c0f04e5..d43af7291d 100644 --- a/tools/windows/tool_setup/idf_setup.iss.inc +++ b/tools/windows/tool_setup/idf_setup.iss.inc @@ -54,7 +54,7 @@ begin if IDFZIPFileVersion <> '' then begin Url := 'https://github.com/espressif/esp-idf/releases/download/' + IDFZIPFileVersion + '/esp-idf-' + IDFZIPFileVersion + '.zip'; - MirrorUrl := 'https://dl.espressif.com/dl/esp-idf/releases/esp-idf-' + IDFZIPFileVersion + '.zip'; + MirrorUrl := 'https://dl.espressif.com/github_assets/espressif/esp-idf/releases/download/' + IDFZIPFileVersion + '/esp-idf-' + IDFZIPFileVersion + '.zip'; IDFZIPFileName := ExpandConstant('{app}\releases\esp-idf-' + IDFZIPFileVersion + '.zip') if not FileExists(IDFZIPFileName) then begin From 0f3595b1e84dabde1a12bfc229e66f94f930dd25 Mon Sep 17 00:00:00 2001 From: daiziyan Date: Thu, 3 Dec 2020 15:26:17 +0800 Subject: [PATCH 5/5] add CN translation for index.rst in get started section for MR10616 --- docs/zh_CN/get-started/index.rst | 39 +++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/docs/zh_CN/get-started/index.rst b/docs/zh_CN/get-started/index.rst index c3a26f4c86..98e10aace7 100644 --- a/docs/zh_CN/get-started/index.rst +++ b/docs/zh_CN/get-started/index.rst @@ -10,6 +10,8 @@ :link_to_translation:`en:[English]` +.. 请保证 README.md 文件与该文件保持同步 + 本文档旨在指导用户搭建 {IDF_TARGET_NAME} 硬件开发的软件环境，通过一个简单的示例展示如何使用 ESP-IDF (Espressif IoT Development Framework) 配置菜单，并编译、下载固件至 {IDF_TARGET_NAME} 开发板等步骤。 .. include-build-file:: inc/version-note.inc @@ -17,7 +19,7 @@ 概述 ============ -.. only:esp32 +.. only:: esp32 ESP32 SoC 芯片支持以下功能： @@ -27,7 +29,7 @@ * 超低功耗协处理器 * 多种外设 -.. only:esp32s2 +.. only:: esp32s2 ESP32-S2 SoC 芯片支持以下功能： @@ -225,6 +227,37 @@ Linux 和 macOS 操作系统 cd ~/esp/esp-idf ./install.sh + +下载工具备选方案 +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +ESP-IDF 工具安装器会下载 Github 发布版本中附带的一些工具，如果访问 Github 较为缓慢，则可以设置一个环境变量，实现优先选择 Espressif 的下载服务器进行 Github 资源下载。 + +.. 注解:: 该设置只影响从 Github 发布版本中下载的单个工具，它并不会改变访问任何 Git 仓库的 URL。 + +Windows 操作系统 +----------------- + +如果希望在运行 ESP-IDF 工具安装器或在使用命令行安装工具时优先选择 Espressif 下载服务器，可通过以下方式设置：打开系统控制面板，然后点击高级设置，添加一个新的环境变量（类型为用户或系统都可以，名称为 ``IDF_GITHUB_ASSETS``，值为 ``dl.espressif.com/github_assets``），最后点击确定。 + +如果在添加新的环境变量前命令行窗口或 ESP-IDF 工具安装器窗口已经打开，请关闭这些窗口后重新打开。 + +当设置好这个新的环境变量后，ESP-IDF 工具安装器以及命令行安装程序将会优先选择 Espressif 下载服务器。 + +.. 在 ESP-IDF 工具安装器的二进制文件更新后（导入复选框），这段需要重新更新 + +Linux 和 macOS 操作系统 +-------------------------- + +要在安装工具时优先选择 Espressif 下载服务器，请在运行 ``install.sh`` 时使用以下命令： + +.. code-block:: bash + + cd ~/esp/esp-idf + export IDF_GITHUB_ASSETS="dl.espressif.com/github_assets" + ./install.sh + + 自定义工具安装路径 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -649,4 +682,4 @@ Windows 操作系统 :esp32: ../get-started-legacy/index .. _Stable version: https://docs.espressif.com/projects/esp-idf/zh_CN/stable/ -.. _Releases page: https://github.com/espressif/esp-idf/releases +.. _Releases page: https://github.com/espressif/esp-idf/releases \ No newline at end of file