From fcf76320c84f92c37379f72024b226af3d38b477 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Mon, 11 Nov 2019 11:39:42 +0800 Subject: [PATCH] docs: Start refactoring IDF-specific docs features into extensions Run the actual IDF build system to determine what components are linked for a particular target. --- docs/conf_common.py | 70 +++------------------------- docs/idf_build_system/CMakeLists.txt | 6 +++ docs/idf_build_system/__init__.py | 48 +++++++++++++++++++ docs/kconfig_reference.py | 52 +++++++++++++++++++++ 4 files changed, 112 insertions(+), 64 deletions(-) create mode 100644 docs/idf_build_system/CMakeLists.txt create mode 100644 docs/idf_build_system/__init__.py create mode 100644 docs/kconfig_reference.py diff --git a/docs/conf_common.py b/docs/conf_common.py index 84cf86a3b1..4622eccb4d 100644 --- a/docs/conf_common.py +++ b/docs/conf_common.py @@ -35,12 +35,16 @@ try: except KeyError: builddir = '_build' +builddir = os.path.abspath(builddir) + # Fill in a default IDF_PATH if it's missing (ie when Read The Docs is building the docs) try: idf_path = os.environ['IDF_PATH'] except KeyError: idf_path = os.path.realpath(os.path.join(os.path.dirname(__file__), '..')) +# Set the idf_target chip. This is a hack right now. +idf_target = 'esp32s2' def call_with_python(cmd): # using sys.executable ensures that the scripts are called with the same Python interpreter @@ -60,70 +64,6 @@ copy_if_modified('xml/', 'xml_in/') # Generate 'api_name.inc' files using the XML files by Doxygen call_with_python('../gen-dxd.py') - -def find_component_files(parent_dir, target_filename): - parent_dir = os.path.abspath(parent_dir) - result = [] - - component_files = dict() - - for (dirpath, dirnames, filenames) in os.walk(parent_dir): - try: - # note: trimming "examples" dir as MQTT submodule - # has its own examples directory in the submodule, not part of IDF - dirnames.remove("examples") - except ValueError: - pass - if target_filename in filenames: - component_files[os.path.basename(dirpath)] = os.path.join(dirpath, target_filename) - - components = sorted(component_files.keys()) - - for component in components: - result.append(component_files[component]) - - print("List of %s: %s" % (target_filename, ", ".join(components))) - return result - - -# Generate 'kconfig.inc' file from components' Kconfig files -print("Generating kconfig.inc from kconfig contents") -kconfig_inc_path = '{}/inc/kconfig.inc'.format(builddir) -temp_sdkconfig_path = '{}/sdkconfig.tmp'.format(builddir) - -kconfigs = find_component_files("../../components", "Kconfig") -kconfig_projbuilds = find_component_files("../../components", "Kconfig.projbuild") -sdkconfig_renames = find_component_files("../../components", "sdkconfig.rename") - -kconfigs_source_path = '{}/inc/kconfigs_source.in'.format(builddir) -kconfig_projbuilds_source_path = '{}/inc/kconfig_projbuilds_source.in'.format(builddir) - -prepare_kconfig_files_args = [sys.executable, - "../../tools/kconfig_new/prepare_kconfig_files.py", - "--env", "COMPONENT_KCONFIGS={}".format(" ".join(kconfigs)), - "--env", "COMPONENT_KCONFIGS_PROJBUILD={}".format(" ".join(kconfig_projbuilds)), - "--env", "COMPONENT_KCONFIGS_SOURCE_FILE={}".format(kconfigs_source_path), - "--env", "COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE={}".format(kconfig_projbuilds_source_path), - ] -subprocess.check_call(prepare_kconfig_files_args) - -confgen_args = [sys.executable, - "../../tools/kconfig_new/confgen.py", - "--kconfig", "../../Kconfig", - "--sdkconfig-rename", "../../sdkconfig.rename", - "--config", temp_sdkconfig_path, - "--env", "COMPONENT_KCONFIGS={}".format(" ".join(kconfigs)), - "--env", "COMPONENT_KCONFIGS_PROJBUILD={}".format(" ".join(kconfig_projbuilds)), - "--env", "COMPONENT_SDKCONFIG_RENAMES={}".format(" ".join(sdkconfig_renames)), - "--env", "COMPONENT_KCONFIGS_SOURCE_FILE={}".format(kconfigs_source_path), - "--env", "COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE={}".format(kconfig_projbuilds_source_path), - "--env", "IDF_PATH={}".format(idf_path), - "--env", "IDF_TARGET={}".format(os.environ.get('IDF_TARGET', 'esp32')), - "--output", "docs", kconfig_inc_path + '.in' - ] -subprocess.check_call(confgen_args) -copy_if_modified(kconfig_inc_path + '.in', kconfig_inc_path) - # Generate 'esp_err_defs.inc' file with ESP_ERR_ error code definitions esp_err_inc_path = '{}/inc/esp_err_defs.inc'.format(builddir) call_with_python('../../tools/gen_esp_err_to_name.py --rst_output ' + esp_err_inc_path + '.in') @@ -176,6 +116,8 @@ extensions = ['breathe', 'sphinxcontrib.rackdiag', 'sphinxcontrib.packetdiag', 'html_redirects', + 'idf_build_system', + 'kconfig_reference', 'sphinx.ext.todo', ] diff --git a/docs/idf_build_system/CMakeLists.txt b/docs/idf_build_system/CMakeLists.txt new file mode 100644 index 0000000000..32ab221c98 --- /dev/null +++ b/docs/idf_build_system/CMakeLists.txt @@ -0,0 +1,6 @@ +# The following lines of boilerplate have to be in your project's +# CMakeLists in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.5) + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(dummy_project) diff --git a/docs/idf_build_system/__init__.py b/docs/idf_build_system/__init__.py new file mode 100644 index 0000000000..db57103e40 --- /dev/null +++ b/docs/idf_build_system/__init__.py @@ -0,0 +1,48 @@ +# Sphinx extension to integrate IDF build system information +# into the Sphinx Build +# +# Runs early in the Sphinx process, runs CMake to generate the dummy IDF project +# in this directory - including resolving paths, etc. +# +# Then emits the new 'idf-info' event which has information read from IDF +# build system, that other extensions can use to generate relevant data. +import os.path +import sys +import subprocess +import json + +# this directory also contains the dummy IDF project +project_path = os.path.abspath(os.path.dirname(__file__)) +project_build_dir = os.path.join(project_path, "build") + +def setup(app): + builddir = os.path.dirname(app.doctreedir.rstrip(os.sep)) + app.add_config_value('idf_path', os.environ.get("IDF_PATH", ""), 'env') + app.add_config_value('idf_target', 'esp32', 'env') + app.add_event('idf-info') + + # Attaching the generate event to env-get-outdated is a bit of a hack, + # we want this to run early in the docs build but unclear exactly when + app.connect('env-get-outdated', generate_idf_info) + +def generate_idf_info(app, env, added, changed, removed): + print("Running CMake on dummy project to get build info...") + idf_py_path = os.path.join(app.config.idf_path, "tools", "idf.py") + print("Running idf.py...") + subprocess.check_call([sys.executable, + idf_py_path, + "-C", + project_path, + "set-target", + app.config.idf_target]) + # TODO: can call these in one execution pass? + subprocess.check_call([sys.executable, + idf_py_path, + "-C", + project_path, + "reconfigure"]) + with open(os.path.join(project_build_dir, "project_description.json")) as f: + project_description = json.load(f) + app.emit('idf-info', project_description) + + return [] diff --git a/docs/kconfig_reference.py b/docs/kconfig_reference.py new file mode 100644 index 0000000000..3c172bd544 --- /dev/null +++ b/docs/kconfig_reference.py @@ -0,0 +1,52 @@ +# Extension to generate the KConfig reference list +import os.path +import sys +import subprocess + +from local_util import copy_if_modified + +def setup(app): + # The idf_build_system extension will emit this event once it + # has parsed the IDF project's information + app.connect('idf-info', generate_reference) + +def generate_reference(app, project_description): + build_dir = os.path.dirname(app.doctreedir.rstrip(os.sep)) + + # Generate 'kconfig.inc' file from components' Kconfig files + print("Generating kconfig.inc from kconfig contents") + kconfig_inc_path = '{}/inc/kconfig.inc'.format(build_dir) + temp_sdkconfig_path = '{}/sdkconfig.tmp'.format(build_dir) + + kconfigs = project_description["config_environment"]["COMPONENT_KCONFIGS"].split(";") + kconfig_projbuilds = project_description["config_environment"]["COMPONENT_KCONFIGS_PROJBUILD"].split(";") + + sdkconfig_renames = set() + # TODO: this should be generated in project description as well, if possible + for k in kconfigs + kconfig_projbuilds: + component_dir = os.path.dirname(k) + sdkconfig_rename = os.path.join(component_dir, "sdkconfig.rename") + if os.path.exists(sdkconfig_rename): + sdkconfig_renames.add(sdkconfig_rename) + + kconfigs_source_path = '{}/inc/kconfigs_source.in'.format(build_dir) + kconfig_projbuilds_source_path = '{}/inc/kconfig_projbuilds_source.in'.format(build_dir) + + confgen_args = [sys.executable, + "../../tools/kconfig_new/confgen.py", + "--kconfig", "../../Kconfig", + "--sdkconfig-rename", "../../sdkconfig.rename", + "--config", temp_sdkconfig_path, + "--env", "COMPONENT_KCONFIGS={}".format(" ".join(kconfigs)), + "--env", "COMPONENT_KCONFIGS_PROJBUILD={}".format(" ".join(kconfig_projbuilds)), + "--env", "COMPONENT_SDKCONFIG_RENAMES={}".format(" ".join(sdkconfig_renames)), + "--env", "COMPONENT_KCONFIGS_SOURCE_FILE={}".format(kconfigs_source_path), + "--env", "COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE={}".format(kconfig_projbuilds_source_path), + "--env", "IDF_PATH={}".format(app.config.idf_path), + "--env", "IDF_TARGET={}".format(app.config.idf_target), + "--output", "docs", kconfig_inc_path + '.in' + ] + subprocess.check_call(confgen_args) + copy_if_modified(kconfig_inc_path + '.in', kconfig_inc_path) + +