From 022a1da4e9e05a752159bcb211763236510c9f17 Mon Sep 17 00:00:00 2001 From: Renz Christian Bagaporo Date: Tue, 5 Feb 2019 11:05:16 +0800 Subject: [PATCH 1/4] ldgen: create python script to find linker script includes --- tools/ci/executable-list.txt | 1 + tools/ldgen/lddeps.py | 56 ++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 tools/ldgen/lddeps.py diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 06821ebbd9..ba84bd2134 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -66,6 +66,7 @@ tools/ci/multirun_with_pyenv.sh components/espcoredump/test/test_espcoredump.py components/espcoredump/test/test_espcoredump.sh tools/ldgen/ldgen.py +tools/ldgen/lddeps.py tools/ldgen/test/test_fragments.py tools/ldgen/test/test_generation.py examples/build_system/cmake/idf_as_lib/build.sh diff --git a/tools/ldgen/lddeps.py b/tools/ldgen/lddeps.py new file mode 100644 index 0000000000..f3963955a9 --- /dev/null +++ b/tools/ldgen/lddeps.py @@ -0,0 +1,56 @@ + +# !/usr/bin/env python +# +# Copyright 2019 Espressif Systems (Shanghai) PTE LTD +# +# This script is used by the linker script generation in order to list a template's +# INCLUDE'd linker scripts recursively. This is so that updates to these INCLUDE'd +# scripts also trigger a relink of the app. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +import re +import argparse + + +INCLUDE_PATTERN = re.compile(r"INCLUDE[\t ]+([^\n]+)") + + +def find_includes(file_path, includes_list): + # Find include files recursively + file_dir = os.path.dirname(file_path) + + with open(file_path, "r") as f: + includes_list.append(file_path) + includes = re.findall(INCLUDE_PATTERN, f.read()) + + for include in includes: + include_file_path = os.path.abspath(os.path.join(file_dir, include)) + find_includes(include_file_path, includes_list) + + +def main(): + argparser = argparse.ArgumentParser(description="List INCLUDE'd linker scripts recursively") + argparser.add_argument("input", help="input linker script") + args = argparser.parse_args() + + includes_list = list() + find_includes(args.input, includes_list) + includes_list.remove(args.input) + + print(" ".join(includes_list)) + + +if __name__ == "__main__": + main() From acd0be823993d3a9c6f5107158dbd53fe18ca261 Mon Sep 17 00:00:00 2001 From: Renz Christian Bagaporo Date: Wed, 19 Dec 2018 22:16:58 +0800 Subject: [PATCH 2/4] cmake,make: add dependencies on template included scripts --- make/common.mk | 2 ++ make/ldgen.mk | 6 ++++-- make/project.mk | 1 - tools/cmake/ldgen.cmake | 15 ++++++++++++++- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/make/common.mk b/make/common.mk index 2eb0f0d3c0..468bf2c688 100644 --- a/make/common.mk +++ b/make/common.mk @@ -2,6 +2,8 @@ # and component makefiles (component_wrapper.mk) # +PYTHON=$(call dequote,$(CONFIG_PYTHON)) + # Include project config makefile, if it exists. # # (Note that we only rebuild this makefile automatically for some diff --git a/make/ldgen.mk b/make/ldgen.mk index 589df4db9b..244de16904 100644 --- a/make/ldgen.mk +++ b/make/ldgen.mk @@ -18,7 +18,8 @@ define ldgen_process_template $(BUILD_DIR_BASE)/ldgen.section_infos: $(LDGEN_SECTIONS_INFO_FILES) $(IDF_PATH)/make/ldgen.mk printf "$(foreach info,$(LDGEN_SECTIONS_INFO_FILES),$(subst \,/,$(shell cygpath -w $(info)))\n)" > $(BUILD_DIR_BASE)/ldgen.section_infos -$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen.section_infos +$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen.section_infos \ +$(shell $(PYTHON) $(IDF_PATH)/tools/ldgen/lddeps.py $(abspath $(1))) @echo 'Generating $(notdir $(2))' $(PYTHON) $(IDF_PATH)/tools/ldgen/ldgen.py \ --input $(1) \ @@ -36,7 +37,8 @@ define ldgen_process_template $(BUILD_DIR_BASE)/ldgen.section_infos: $(LDGEN_SECTIONS_INFO_FILES) $(IDF_PATH)/make/ldgen.mk printf "$(foreach info,$(LDGEN_SECTIONS_INFO_FILES),$(info)\n)" > $(BUILD_DIR_BASE)/ldgen.section_infos -$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen.section_infos +$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen.section_infos \ +$(shell $(PYTHON) $(IDF_PATH)/tools/ldgen/lddeps.py $(abspath $(1))) @echo 'Generating $(notdir $(2))' $(PYTHON) $(IDF_PATH)/tools/ldgen/ldgen.py \ --input $(1) \ diff --git a/make/project.mk b/make/project.mk index cbd93b346c..ed9e535b0d 100644 --- a/make/project.mk +++ b/make/project.mk @@ -417,7 +417,6 @@ export COMPILER_VERSION_STR COMPILER_VERSION_NUM GCC_NOT_5_2_0 CPPFLAGS += -DGCC_NOT_5_2_0=$(GCC_NOT_5_2_0) export CPPFLAGS -PYTHON=$(call dequote,$(CONFIG_PYTHON)) # the app is the main executable built by the project APP_ELF:=$(BUILD_DIR_BASE)/$(PROJECT_NAME).elf diff --git a/tools/cmake/ldgen.cmake b/tools/cmake/ldgen.cmake index b8e58c0ee1..6ea6d5d830 100644 --- a/tools/cmake/ldgen.cmake +++ b/tools/cmake/ldgen.cmake @@ -47,6 +47,18 @@ endfunction() # Passes a linker script template to the linker script generation tool for # processing function(ldgen_process_template template output) + get_filename_component(template ${template} ABSOLUTE BASE_DIR ${CMAKE_CURRENT_LIST_DIR}) + execute_process(COMMAND ${PYTHON} ${IDF_PATH}/tools/ldgen/lddeps.py ${template} + OUTPUT_VARIABLE template_includes + ERROR_VARIABLE template_includes_err + ) + + if(template_includes_err) + message(FATAL_ERROR "Unable to parse linker script template for INCLUDEs\n" ${template_includes_err}) + endif() + + spaces2list(template_includes) + file(GENERATE OUTPUT ${CMAKE_BINARY_DIR}/ldgen.section_infos CONTENT "$,\n>") @@ -65,7 +77,8 @@ function(ldgen_process_template template output) --env "IDF_CMAKE=y" --env "IDF_PATH=${IDF_PATH}" --env "IDF_TARGET=${IDF_TARGET}" - DEPENDS ${template} $ ${SDKCONFIG} ldgen_section_infos + DEPENDS ${template} $ ${SDKCONFIG} + ldgen_section_infos ${template_includes} ) get_filename_component(output_name ${output} NAME) From 50e860fe8cb1e436eaae5da5ed21b5097c42c5cd Mon Sep 17 00:00:00 2001 From: Renz Christian Bagaporo Date: Thu, 20 Dec 2018 09:30:01 +0800 Subject: [PATCH 3/4] ci: test relink on template included file change --- tools/ci/test_build_system.sh | 7 +++++++ tools/ci/test_build_system_cmake.sh | 10 ++++++++++ 2 files changed, 17 insertions(+) diff --git a/tools/ci/test_build_system.sh b/tools/ci/test_build_system.sh index 99da318d61..4c49de625f 100755 --- a/tools/ci/test_build_system.sh +++ b/tools/ci/test_build_system.sh @@ -171,6 +171,13 @@ function run_tests() assert_rebuilt ${APP_BINS} assert_not_rebuilt ${BOOTLOADER_BINS} + print_status "Touching linker script included in template should re-link app" + take_build_snapshot + touch ${IDF_PATH}/components/esp32/ld/esp32.spiram.rom-functions-iram.ld + make + assert_rebuilt ${APP_BINS} + assert_not_rebuilt ${BOOTLOADER_BINS} + print_status "sdkconfig update triggers full recompile" make take_build_snapshot diff --git a/tools/ci/test_build_system_cmake.sh b/tools/ci/test_build_system_cmake.sh index 3fb5cb103d..48ef8c1857 100755 --- a/tools/ci/test_build_system_cmake.sh +++ b/tools/ci/test_build_system_cmake.sh @@ -186,6 +186,16 @@ function run_tests() assert_not_rebuilt ${BOOTLOADER_BINS} mv esp32_fragments.lf ${IDF_PATH}/components/esp32/ld/ + print_status "Updating linker script included in template should re-link app" + take_build_snapshot + cp ${IDF_PATH}/components/esp32/ld/esp32.spiram.rom-functions-iram.ld . + sleep 1 # ninja may ignore if the timestamp delta is too low + echo "/* (Build test comment) */" >> ${IDF_PATH}/components/esp32/ld/esp32.spiram.rom-functions-iram.ld + idf.py build || failure "Failed to build with modified linker script included in template" + assert_rebuilt ${APP_BINS} + assert_not_rebuilt ${BOOTLOADER_BINS} + mv esp32.spiram.rom-functions-iram.ld ${IDF_PATH}/components/esp32/ld/ + print_status "sdkconfig update triggers full recompile" clean_build_dir idf.py build From 6b7f4bc72f8bdf9bb8557b7300794512f321456c Mon Sep 17 00:00:00 2001 From: Renz Christian Bagaporo Date: Thu, 14 Feb 2019 18:51:35 +0800 Subject: [PATCH 4/4] make: remove unecessary inclusion of ldgen.mk --- make/component_wrapper.mk | 2 -- 1 file changed, 2 deletions(-) diff --git a/make/component_wrapper.mk b/make/component_wrapper.mk index 449ad9fc73..049a8084b3 100644 --- a/make/component_wrapper.mk +++ b/make/component_wrapper.mk @@ -155,8 +155,6 @@ OWN_INCLUDES:=$(abspath $(addprefix $(COMPONENT_PATH)/,$(COMPONENT_PRIV_INCLUDED COMPONENT_INCLUDES := $(OWN_INCLUDES) $(filter-out $(OWN_INCLUDES),$(COMPONENT_INCLUDES)) -include $(IDF_PATH)/make/ldgen.mk - ################################################################################ # 4) Define a target to generate component_project_vars.mk Makefile which # contains common per-component settings which are included directly in the