Merge branch 'bugfix/relink_on_included_linker_script_change' into 'master'

Re-link when linker script included from template file are modified

See merge request idf/esp-idf!4001

make/common.mk

@@ -2,6 +2,8 @@
 # and component makefiles (component_wrapper.mk)
 #
 
+PYTHON=$(call dequote,$(CONFIG_PYTHON))
+
 # Include project config makefile, if it exists.
 #
 # (Note that we only rebuild this makefile automatically for some

make/component_wrapper.mk

@@ -155,8 +155,6 @@ OWN_INCLUDES:=$(abspath $(addprefix $(COMPONENT_PATH)/,$(COMPONENT_PRIV_INCLUDED
 COMPONENT_INCLUDES := $(OWN_INCLUDES) $(filter-out $(OWN_INCLUDES),$(COMPONENT_INCLUDES))
 
 
-include $(IDF_PATH)/make/ldgen.mk
-
 ################################################################################
 # 4) Define a target to generate component_project_vars.mk Makefile which
 # contains common per-component settings which are included directly in the

make/ldgen.mk

@@ -18,7 +18,8 @@ define ldgen_process_template
 $(BUILD_DIR_BASE)/ldgen.section_infos: $(LDGEN_SECTIONS_INFO_FILES) $(IDF_PATH)/make/ldgen.mk
 	printf "$(foreach info,$(LDGEN_SECTIONS_INFO_FILES),$(subst \,/,$(shell cygpath -w $(info)))\n)" > $(BUILD_DIR_BASE)/ldgen.section_infos
 
-$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen.section_infos
+$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen.section_infos \
+$(shell $(PYTHON) $(IDF_PATH)/tools/ldgen/lddeps.py $(abspath $(1)))
 	@echo 'Generating $(notdir $(2))'
 	$(PYTHON) $(IDF_PATH)/tools/ldgen/ldgen.py \
 		--input         $(1) \
@@ -36,7 +37,8 @@ define ldgen_process_template
 $(BUILD_DIR_BASE)/ldgen.section_infos: $(LDGEN_SECTIONS_INFO_FILES) $(IDF_PATH)/make/ldgen.mk
 	printf "$(foreach info,$(LDGEN_SECTIONS_INFO_FILES),$(info)\n)" > $(BUILD_DIR_BASE)/ldgen.section_infos
 
-$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen.section_infos
+$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen.section_infos \
+$(shell $(PYTHON) $(IDF_PATH)/tools/ldgen/lddeps.py $(abspath $(1)))
 	@echo 'Generating $(notdir $(2))'
 	$(PYTHON) $(IDF_PATH)/tools/ldgen/ldgen.py \
 		--input         $(1) \

make/project.mk

@@ -417,7 +417,6 @@ export COMPILER_VERSION_STR COMPILER_VERSION_NUM GCC_NOT_5_2_0
 CPPFLAGS += -DGCC_NOT_5_2_0=$(GCC_NOT_5_2_0)
 export CPPFLAGS
 
-PYTHON=$(call dequote,$(CONFIG_PYTHON))
 
 # the app is the main executable built by the project
 APP_ELF:=$(BUILD_DIR_BASE)/$(PROJECT_NAME).elf

tools/ci/executable-list.txt

@@ -66,6 +66,7 @@ tools/ci/multirun_with_pyenv.sh
 components/espcoredump/test/test_espcoredump.py
 components/espcoredump/test/test_espcoredump.sh
 tools/ldgen/ldgen.py
+tools/ldgen/lddeps.py
 tools/ldgen/test/test_fragments.py
 tools/ldgen/test/test_generation.py
 examples/build_system/cmake/idf_as_lib/build.sh

tools/ci/test_build_system.sh

@@ -171,6 +171,13 @@ function run_tests()
     assert_rebuilt ${APP_BINS}
     assert_not_rebuilt ${BOOTLOADER_BINS}
 
+    print_status "Touching linker script included in template should re-link app"
+    take_build_snapshot
+    touch ${IDF_PATH}/components/esp32/ld/esp32.spiram.rom-functions-iram.ld
+    make
+    assert_rebuilt ${APP_BINS}
+    assert_not_rebuilt ${BOOTLOADER_BINS}
+
     print_status "sdkconfig update triggers full recompile"
     make
     take_build_snapshot

tools/ci/test_build_system_cmake.sh

@@ -186,6 +186,16 @@ function run_tests()
     assert_not_rebuilt ${BOOTLOADER_BINS}
     mv esp32_fragments.lf ${IDF_PATH}/components/esp32/ld/
 
+    print_status "Updating linker script included in template should re-link app"
+    take_build_snapshot
+    cp ${IDF_PATH}/components/esp32/ld/esp32.spiram.rom-functions-iram.ld .
+    sleep 1  # ninja may ignore if the timestamp delta is too low
+    echo "/* (Build test comment) */" >> ${IDF_PATH}/components/esp32/ld/esp32.spiram.rom-functions-iram.ld
+    idf.py build || failure "Failed to build with modified linker script included in template"
+    assert_rebuilt ${APP_BINS}
+    assert_not_rebuilt ${BOOTLOADER_BINS}
+    mv esp32.spiram.rom-functions-iram.ld ${IDF_PATH}/components/esp32/ld/
+
     print_status "sdkconfig update triggers full recompile"
     clean_build_dir
     idf.py build

tools/cmake/ldgen.cmake

@@ -47,6 +47,18 @@ endfunction()
 # Passes a linker script template to the linker script generation tool for
 # processing
 function(ldgen_process_template template output)
+    get_filename_component(template ${template} ABSOLUTE BASE_DIR ${CMAKE_CURRENT_LIST_DIR})
+    execute_process(COMMAND ${PYTHON} ${IDF_PATH}/tools/ldgen/lddeps.py ${template}
+                    OUTPUT_VARIABLE template_includes
+                    ERROR_VARIABLE template_includes_err
+                    )
+
+    if(template_includes_err)
+        message(FATAL_ERROR "Unable to parse linker script template for INCLUDEs\n" ${template_includes_err})
+    endif()
+
+    spaces2list(template_includes)
+
     file(GENERATE OUTPUT ${CMAKE_BINARY_DIR}/ldgen.section_infos
         CONTENT "$<JOIN:$<TARGET_PROPERTY:ldgen_section_infos,SECTIONS_INFO_FILES>,\n>")
 
@@ -65,7 +77,8 @@ function(ldgen_process_template template output)
         --env       "IDF_CMAKE=y"
         --env       "IDF_PATH=${IDF_PATH}"
         --env       "IDF_TARGET=${IDF_TARGET}"
-        DEPENDS     ${template} $<TARGET_PROPERTY:ldgen,FRAGMENT_FILES> ${SDKCONFIG} ldgen_section_infos
+        DEPENDS     ${template} $<TARGET_PROPERTY:ldgen,FRAGMENT_FILES> ${SDKCONFIG}
+                    ldgen_section_infos ${template_includes}
     )
 
     get_filename_component(output_name ${output} NAME)

tools/ldgen/lddeps.py

@@ -0,0 +1,56 @@
+
+# !/usr/bin/env python
+#
+# Copyright 2019 Espressif Systems (Shanghai) PTE LTD
+#
+# This script is used by the linker script generation in order to list a template's
+# INCLUDE'd linker scripts recursively. This is so that updates to these INCLUDE'd
+# scripts also trigger a relink of the app.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import os
+import re
+import argparse
+
+
+INCLUDE_PATTERN = re.compile(r"INCLUDE[\t ]+([^\n]+)")
+
+
+def find_includes(file_path, includes_list):
+    # Find include files recursively
+    file_dir = os.path.dirname(file_path)
+
+    with open(file_path, "r") as f:
+        includes_list.append(file_path)
+        includes = re.findall(INCLUDE_PATTERN, f.read())
+
+    for include in includes:
+        include_file_path = os.path.abspath(os.path.join(file_dir, include))
+        find_includes(include_file_path, includes_list)
+
+
+def main():
+    argparser = argparse.ArgumentParser(description="List INCLUDE'd linker scripts recursively")
+    argparser.add_argument("input", help="input linker script")
+    args = argparser.parse_args()
+
+    includes_list = list()
+    find_includes(args.input, includes_list)
+    includes_list.remove(args.input)
+
+    print(" ".join(includes_list))
+
+
+if __name__ == "__main__":
+    main()