diff --git a/tools/ci/check_public_headers.py b/tools/ci/check_public_headers.py new file mode 100644 index 0000000000..fe1dfff21a --- /dev/null +++ b/tools/ci/check_public_headers.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python +# +# Checks all public headers in IDF in the ci +# +# Copyright 2020 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import print_function +from __future__ import unicode_literals +import re +import os +import subprocess +import json +import fnmatch +import argparse +import queue +from threading import Thread, Event +import tempfile + + +class HeaderFailed(Exception): + """Base header failure exeption""" + pass + + +class HeaderFailedSdkconfig(HeaderFailed): + def __str__(self): + return "Sdkconfig Error" + + +class HeaderFailedBuildError(HeaderFailed): + def __str__(self): + return "Header Build Error" + + +class HeaderFailedCppGuardMissing(HeaderFailed): + def __str__(self): + return "Header Missing C++ Guard" + + +class HeaderFailedContainsCode(HeaderFailed): + def __str__(self): + return "Header Produced non-zero object" + + +# Creates a temp file and returns both output as a string and a file name +# +def exec_cmd_to_temp_file(what, suffix=""): + out_file = tempfile.NamedTemporaryFile(suffix=suffix, delete=False) + rc, out, err = exec_cmd(what, out_file) + with open(out_file.name, "r") as f: + out = f.read() + return rc, out, err, out_file.name + + +def exec_cmd(what, out_file=subprocess.PIPE): + p = subprocess.Popen(what, stdin=subprocess.PIPE, stdout=out_file, stderr=subprocess.PIPE) + output, err = p.communicate() + rc = p.returncode + output = output.decode('utf-8') if output is not None else None + err = err.decode('utf-8') if err is not None else None + return rc, output, err + + +class PublicHeaderChecker: + # Intermediate results + COMPILE_ERR_REF_CONFIG_HDR_FAILED = 1 # -> Cannot compile and failed with injected SDKCONFIG #error (header FAILs) + COMPILE_ERR_ERROR_MACRO_HDR_OK = 2 # -> Cannot compile, but failed with "#error" directive (header seems OK) + COMPILE_ERR_HDR_FAILED = 3 # -> Cannot compile with another issue, logged if verbose (header FAILs) + PREPROC_OUT_ZERO_HDR_OK = 4 # -> Both preprocessors produce zero out (header file is OK) + PREPROC_OUT_SAME_HRD_FAILED = 5 # -> Both preprocessors produce the same, non-zero output (header file FAILs) + PREPROC_OUT_DIFFERENT_WITH_EXT_C_HDR_OK = 6 # -> Both preprocessors produce different, non-zero output with extern "C" (header seems OK) + PREPROC_OUT_DIFFERENT_NO_EXT_C_HDR_FAILED = 7 # -> Both preprocessors produce different, non-zero output without extern "C" (header fails) + + def log(self, message): + if self.verbose: + print(message) + + def __init__(self, verbose=False, jobs=1, prefix=None): + self.gcc = "{}gcc".format(prefix) + self.gpp = "{}g++".format(prefix) + self.verbose = verbose + self.jobs = jobs + self.prefix = prefix + self.extern_c = re.compile(r'extern "C"') + self.error_macro = re.compile(r'#error') + self.error_orphan_kconfig = re.compile(r"#error CONFIG_VARS_USED_WHILE_SDKCONFIG_NOT_INCLUDED") + self.kconfig_macro = re.compile(r'\bCONFIG_[A-Z0-9_]+') + self.assembly_nocode = r'^\s*(\.file|\.text|\.ident).*$' + self.check_threads = [] + + self.job_queue = queue.Queue() + self.failed_queue = queue.Queue() + self.terminate = Event() + + def __enter__(self): + for i in range(self.jobs): + t = Thread(target=self.check_headers, args=(i, )) + self.check_threads.append(t) + t.start() + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.terminate.set() + for t in self.check_threads: + t.join() + + # thread function process incoming header file from a queue + def check_headers(self, num): + while not self.terminate.is_set(): + if not self.job_queue.empty(): + task = self.job_queue.get() + if task is None: + self.terminate.set() + else: + try: + self.check_one_header(task, num) + except HeaderFailed as e: + self.failed_queue.put("{}: Failed! {}".format(task, e)) + + def get_failed(self): + return list(self.failed_queue.queue) + + def join(self): + for t in self.check_threads: + while t.isAlive and not self.terminate.is_set(): + t.join(1) # joins with timeout to respond to keyboard interrupt + + # Checks one header calling: + # - preprocess_one_header() to test and compare preprocessor outputs + # - check_no_code() to test if header contains some executable code + # Procedure + # 1) Preprocess the include file with C preprocessor and with CPP preprocessor + # - Pass the test if the preprocessor outputs are the same and whitespaces only (#define only header) + # - Fail the test if the preprocessor outputs are the same (but with some code) + # - If outputs different, continue with 2) + # 2) Strip out all include directives to generate "temp.h" + # 3) Preprocess the temp.h the same way in (1) + # - Pass the test if the preprocessor outputs are the same and whitespaces only (#include only header) + # - Fail the test if the preprocessor outputs are the same (but with some code) + # - If outputs different, pass the test + # 4) If header passed the steps 1) and 3) test that it produced zero assembly code + def check_one_header(self, header, num): + res = self.preprocess_one_header(header, num) + if res == self.COMPILE_ERR_REF_CONFIG_HDR_FAILED: + raise HeaderFailedSdkconfig() + elif res == self.COMPILE_ERR_ERROR_MACRO_HDR_OK: + return self.compile_one_header(header) + elif res == self.COMPILE_ERR_HDR_FAILED: + raise HeaderFailedBuildError() + elif res == self.PREPROC_OUT_ZERO_HDR_OK: + return self.compile_one_header(header) + elif res == self.PREPROC_OUT_SAME_HRD_FAILED: + raise HeaderFailedCppGuardMissing() + else: + self.compile_one_header(header) + try: + _, _, _, temp_header = exec_cmd_to_temp_file(["sed", "/#include/d; /#error/d", header], suffix=".h") + res = self.preprocess_one_header(temp_header, num, ignore_sdkconfig_issue=True) + if res == self.PREPROC_OUT_SAME_HRD_FAILED: + raise HeaderFailedCppGuardMissing() + elif res == self.PREPROC_OUT_DIFFERENT_NO_EXT_C_HDR_FAILED: + raise HeaderFailedCppGuardMissing() + finally: + os.unlink(temp_header) + + def compile_one_header(self, header): + rc, out, err = exec_cmd([self.gcc, "-S", "-o-", "-include", header, self.main_c] + self.include_dir_flags) + if rc == 0: + if not re.sub(self.assembly_nocode, '', out, flags=re.M).isspace(): + raise HeaderFailedContainsCode() + return # Header OK: produced zero code + raise HeaderFailedBuildError() + + def preprocess_one_header(self, header, num, ignore_sdkconfig_issue=False): + all_compilation_flags = ["-w", "-P", "-E", "-include", header, self.main_c] + self.include_dir_flags + if not ignore_sdkconfig_issue: + # just strip commnets to check for CONFIG_... macros + rc, out, err = exec_cmd([self.gcc, "-fpreprocessed", "-dD", "-P", "-E", header] + self.include_dir_flags) + if re.search(self.kconfig_macro, out): + # enable defined #error if sdkconfig.h not included + all_compilation_flags.append("-DIDF_CHECK_SDKCONFIG_INCLUDED") + try: + # compile with C++, check for errors, outputs for a temp file + rc, cpp_out, err, cpp_out_file = exec_cmd_to_temp_file([self.gpp, "--std=c++17"] + all_compilation_flags) + if rc != 0: + if re.search(self.error_macro, err): + if re.search(self.error_orphan_kconfig, err): + self.log("{}: CONFIG_VARS_USED_WHILE_SDKCONFIG_NOT_INCLUDED".format(header)) + return self.COMPILE_ERR_REF_CONFIG_HDR_FAILED + self.log("{}: Error directive failure: OK".format(header)) + return self.COMPILE_ERR_ERROR_MACRO_HDR_OK + self.log("{}: FAILED: compilation issue".format(header)) + self.log(err) + return self.COMPILE_ERR_HDR_FAILED + # compile with C compiler, outputs to another temp file + rc, c99_out, err, c99_out_file = exec_cmd_to_temp_file([self.gcc, "--std=c99"] + all_compilation_flags) + if rc != 0: + self.log("{} FAILED should never happen".format(header)) + return self.COMPILE_ERR_HDR_FAILED + # diff the two outputs + rc, diff, err = exec_cmd(["diff", c99_out_file, cpp_out_file]) + if not diff or diff.isspace(): + if not cpp_out or cpp_out.isspace(): + self.log("{} The same, but empty out - OK".format(header)) + return self.PREPROC_OUT_ZERO_HDR_OK + self.log("{} FAILED It is the same!".format(header)) + return self.PREPROC_OUT_SAME_HRD_FAILED + if re.search(self.extern_c, diff): + self.log("{} extern C present - OK".format(header)) + return self.PREPROC_OUT_DIFFERENT_WITH_EXT_C_HDR_OK + self.log("{} Different but no extern C - FAILED".format(header)) + return self.PREPROC_OUT_DIFFERENT_NO_EXT_C_HDR_FAILED + finally: + os.unlink(cpp_out_file) + try: + os.unlink(c99_out_file) + except Exception: + pass + + # Get compilation data from an example to list all public header files + def list_public_headers(self, ignore_dirs, ignore_files): + idf_path = os.getenv('IDF_PATH') + project_dir = os.path.join(idf_path, "examples", "get-started", "blink") + subprocess.check_call(["idf.py", "reconfigure"], cwd=project_dir) + build_commands_json = os.path.join(project_dir, "build", "compile_commands.json") + with open(build_commands_json, "r") as f: + build_command = json.load(f)[0]["command"].split() + include_dir_flags = [] + include_dirs = [] + # process compilation flags (includes and defines) + for item in build_command: + if item.startswith("-I"): + include_dir_flags.append(item) + if "components" in item: + include_dirs.append(item[2:]) # Removing the leading "-I" + if item.startswith("-D"): + include_dir_flags.append(item.replace('\\','')) # removes escaped quotes, eg: -DMBEDTLS_CONFIG_FILE=\\\"mbedtls/esp_config.h\\\" + include_dir_flags.append("-I" + os.path.join(project_dir, "build", "config")) + sdkconfig_h = os.path.join(project_dir, "build", "config", "sdkconfig.h") + # prepares a main_c file for easier sdkconfig checks and avoid compilers warning when compiling headers directly + with open(sdkconfig_h, "a") as f: + f.write("#define IDF_SDKCONFIG_INCLUDED") + main_c = os.path.join(project_dir, "build", "compile.c") + with open(main_c, "w") as f: + f.write("#if defined(IDF_CHECK_SDKCONFIG_INCLUDED) && ! defined(IDF_SDKCONFIG_INCLUDED)\n" + "#error CONFIG_VARS_USED_WHILE_SDKCONFIG_NOT_INCLUDED\n" + "#endif") + # processes public include dirs, removing ignored files + all_include_files = [] + files_to_check = [] + for d in include_dirs: + if os.path.relpath(d, idf_path).startswith(tuple(ignore_dirs)): + self.log("{} - directory ignored".format(d)) + continue + for root, dirnames, filenames in os.walk(d): + for filename in fnmatch.filter(filenames, '*.h'): + all_include_files.append(os.path.join(root, filename)) + self.main_c = main_c + self.include_dir_flags = include_dir_flags + ignore_files = set(ignore_files) + # processes public include files, removing ignored files + for f in all_include_files: + rel_path_file = os.path.relpath(f, idf_path) + if any([os.path.commonprefix([d, rel_path_file]) == d for d in ignore_dirs]): + self.log("{} - file ignored (inside ignore dir)".format(f)) + continue + if rel_path_file in ignore_files: + self.log("{} - file ignored".format(f)) + continue + files_to_check.append(f) + # removes duplicates and places headers to a work queue + for f in set(files_to_check): + self.job_queue.put(f) + self.job_queue.put(None) # to indicate the last job + + +def check_all_headers(): + parser = argparse.ArgumentParser("Public header checker file") + parser.add_argument("--verbose", "-v", help="enables verbose mode", action="store_true") + parser.add_argument("--jobs", "-j", help="number of jobs to run checker", default=1, type=int) + parser.add_argument("--prefix", "-p", help="compiler prefix", default="xtensa-esp32-elf-", type=str) + parser.add_argument("--exclude-file", "-e", help="exception file", default="check_public_headers_exceptions.txt", type=str) + args = parser.parse_args() + + # process excluded files and dirs + exclude_file = os.path.join(os.path.dirname(__file__), args.exclude_file) + with open(exclude_file, "r") as f: + lines = [line.rstrip() for line in f] + ignore_files = [] + ignore_dirs = [] + for line in lines: + if not line or line.isspace() or line.startswith("#"): + continue + if os.path.isdir(line): + ignore_dirs.append(line) + else: + ignore_files.append(line) + + # start header check + with PublicHeaderChecker(args.verbose, args.jobs, args.prefix) as header_check: + header_check.list_public_headers(ignore_dirs, ignore_files) + try: + header_check.join() + failures = header_check.get_failed() + if len(failures) > 0: + for failed in failures: + print(failed) + exit(1) + print("No errors found") + except KeyboardInterrupt: + print("Keyboard interrupt") + + +if __name__ == '__main__': + check_all_headers() diff --git a/tools/ci/check_public_headers.sh b/tools/ci/check_public_headers.sh deleted file mode 100755 index 9d77d020f2..0000000000 --- a/tools/ci/check_public_headers.sh +++ /dev/null @@ -1,391 +0,0 @@ -#!/bin/bash -# -# This script finds all *.h files in public include dirs and checks for -# * compilation issues -# * c++ header guards -# * need for sdkconfig.h inclusion (if CONFIG_ macros referenced) -# -# Script arguments: -# --verbose: enables printing details if compilation fails (verbose is off by default) -# --process-all-files: all files are checked and then excluded, by default the excluded files are skipped and not checked -# --include-maxdepth=X: maximum directory depth for finding include files in include dirs (by default not specified=infinity) -# --jobs=X: run the checker in X separate processes - -set -euo pipefail - -if [ -z "${IDF_PATH:-}" ]; then - echo "Please run export.sh before invoking this script" - exit 1 -fi - -##################### helper functions ##################### - -# log_error -# -# Logs details on filename processing when the include fails -function log_error() -{ - if [ ! -z "${VERBOSE:-}" ]; then - file=$1 - echo " verbose..." - echo "$LINE" - grep -A5 "error:" - echo "$LINE" - echo -n "$file" - fi -} - -# log_details -# -# Logs details as function argument to stdout -function log_details() -{ - if [ ! -z "${VERBOSE:-}" ]; then - printf "... $1 " - fi -} - - -# preproc_one_header -# -# Function to preprocess a header with CC and CXX and compares result -# Returns -# 0 -> Never returns zero (for easier return value expansion) -# 1 -> Cannot compile and failed with injected SDKCONFIG #error (header FAILs) -# 2 -> Cannot compile, but failed with "#error" directive (header seems OK) -# 3 -> Cannot compile with another issue, logged if verbose (header FAILs) -# 4 -> Both preprocessors produce zero out (header file is OK) -# 5 -> Both preprocessors produce the same, non-zero output (header file FAILs) -# 6 -> Both preprocessors produce different, non-zero output with extern "C" (header seems OK) -# 7 -> Both preprocessors produce different, non-zero output without extern "C" (header fails) -function preproc_one_header() -{ - local extra_flags="" - local file=$1 - # check if the header (after removing comments) references CONFIG_XXX macros - if ${CC} -fpreprocessed -dD -P -E ${file} ${FLAGS_INCLUDE_DIRS} 2>/dev/null | grep -q -wo -E 'CONFIG_[A-Z0-9_]+'; then - # enable sdkconfig.h inclusion checker - extra_flags="-DIDF_CHECK_SDKCONFIG_INCLUDED" - fi - if ! cpp_out=`${CXX} -w -P -E --std=c++17 -include ${file} ${extra_flags} ${TEST_MAIN_C} ${FLAGS_INCLUDE_DIRS} 2>&1`; then - if [[ "$cpp_out" =~ "#error" ]]; then - if [[ "$cpp_out" =~ "#error CONFIG_VARS_USED_WHILE_SDKCONFIG_NOT_INCLUDED" ]]; then - # failed for sdkconfig.h issue - return 1 - fi; - # header passed in general (error directive) - return 2 - fi; - # Logging the compilation issue - echo "$cpp_out" | log_error $file - return 3 - fi; - if ! c99_out=`${CC} -w -P -E --std=c99 -include ${file} ${extra_flags} ${TEST_MAIN_C} ${FLAGS_INCLUDE_DIRS} 2>/dev/null`; then - # Should not happen, fail with compilation issue - return 3 - fi; - if diff_out=`diff <(echo "$cpp_out" ) <(echo "$c99_out")`; then - # checks zero output: contains spaces only if lines < 5 (for performace) - lines=`echo "$cpp_out" | wc -l` - if [ $lines -lt 5 ] && [ -z "${cpp_out// }" ]; then - return 4 - fi - return 5 - elif echo $diff_out | grep -q 'extern "C"'; then - return 6 - fi - return 7 -} - -# check_one_header -# -# Function used to implement the logic of checking a single header. -# Returns 0 on success (as a normal bash function would). -# - returns 1 if suspects C++ guard missing -# - returns 2 if compile issue (other then error directive) -# - returns 3 if CONFIG_XXX variable used but sdkconfig.h not included -# Ideally shouldn't print anything (if verbose, preproc_one_header outputs compilation errors) -function check_one_header -{ - include_file=$1 - # Procedure: - # 1) Preprocess the include file with C preprocessor and with CPP preprocessor - # - Pass the test if the preprocessor outputs are the same and whitespaces only (#define only header) - # - Fail the test if the preprocessor outputs are the same (but with some code) - # - If outputs different, continue with 2) - # 2) Strip out all include directives to generate "temp.h" - # 3) Preprocess the temp.h the same way in (1) - # - Pass the test if the preprocessor outputs are the same and whitespaces only (#include only header) - # - Fail the test if the preprocessor outputs are the same (but with some code) - # - If outputs different, pass the test - # - preproc_one_header ${include_file} || ret=$? - case $ret in - 1*) log_details "sdkconfig issue"; return 3 ;; - 2*) log_details "compile failed with #error"; return 0 ;; - 3*) log_details "another compile issue"; return 2 ;; - 4*) log_details "zero output"; return 0 ;; - 5*) log_details "the same non-zero out"; return 1 ;; - [6-7]*) log_details "different and non-zero: check again without includes"; - # Removing includes to check if the header itself contains a code - temp=$(mktemp) - sed '/#include/d; /#error/d' ${include_file} > ${temp} - case `preproc_one_header ${temp} || echo $?` in - [1-3]*) log_details "ok, compile issue"; return 0 ;; - 4*) log_details "ok, zero output"; return 0 ;; - 5*) log_details "the same non-zero out"; return 1 ;; - 6*) log_details "different, non-zero, with extern \"C\""; return 0 ;; - 7*) log_details "different, non-zero, no extern \"C\""; return 1 ;; - esac - rm -f ${temp} - esac -} - -# create_test_main -# -# Generates a compilation unit to fail if sdkconfig.h inclusion to be checked -# Exports the full path to the file as TEST_MAIN_C -function create_test_main -{ - echo "#if defined(IDF_CHECK_SDKCONFIG_INCLUDED) && ! defined(IDF_SDKCONFIG_INCLUDED) - #error CONFIG_VARS_USED_WHILE_SDKCONFIG_NOT_INCLUDED - #endif" > ${1}/compile.c - export TEST_MAIN_C=${1}/compile.c -} - -# get_include_dir_list -# -# Get the list of public include directories from compile_commands.json. -# Stores the result in INCLUDE_DIRS variable. -function get_include_dirs_list -{ - local proj_dir=${1} - local sdkinclude_dir=${proj_dir}/build/config - pushd $proj_dir - idf.py reconfigure - - # Get the compiler command line from compile_commands.json - # Poor man's replacement of `jq` in Python. - extract_build_command_py='import json; print(json.load(open("build/compile_commands.json", "r"))[0]["command"])' - build_command=$(python -c "${extract_build_command_py}") - - # build the list of include directories, only considering the directories in "components/" - include_dirs="" - for token in ${build_command}; do - if [[ "${token}" =~ ^-I.* ]]; then - flags_include_dirs+=" $token" - fi - if [[ ! "${token}" =~ ^-I.*/components/.* ]]; then - continue - fi - newline=$'\n' - include_dirs="${include_dirs}${token##-I}${newline}" - done - - # Append a config value used to check if the sdkconfig included - echo "#define IDF_SDKCONFIG_INCLUDED" >> ${sdkinclude_dir}/sdkconfig.h - export INCLUDE_DIRS=${include_dirs} - flags_include_dirs+=" -I ${sdkinclude_dir}" - export FLAGS_INCLUDE_DIRS=${flags_include_dirs} -} - -# Utility convertion functions used for passing load sequences to processes (in form of one character) -# -# Returns ordinal number of supplied character -ord() { LC_CTYPE=C printf %d "'$1"; } -# Returns a character represented by ASCII code -chr() { printf "\\$(printf "%o" $1)"; } - -# check_headers_in_process -# -# Checks one include files at a time in a batch of headers referenced by indices from stdin -# This function may be called from separate processes (used simple load balancing) -# Periodically checks files from -# Outputs failed files to -function check_headers_in_process -{ - local proc="$1" - local list_of_includes="$2" - local list_of_failed_includes="$3" - - list=(`cat ${list_of_includes}`) - size=${#list[@]} - let batch=$size/$JOBS_LOAD_CHAR_SIZE+1 - - # Reads one char (due to atomicity) from stdin - # -ord(char) signifies an index to the list of files to be processed - while IFS= read -n1 char; do - batch_index=$(ord $char) - let run=$batch_index-$JOBS_LOAD_CHAR_FROM || true - for i in `seq 1 $batch`; do - let file_index=$run*$batch+$i-1 || true - if [ "$file_index" -lt "$size" ]; then - # log_details "$proc: Processing $file_index (batch=$batch, run=$run)\n" - include_file=${list[$file_index]} - rel_inc_file=${include_file##${IDF_PATH}} - printf "Processing ${rel_inc_file}" - check_one_header ${include_file} && ret=0 || ret=$? - case "$ret" in - 0*) printf " OK\n" ;; - 1*) printf " FAIL (missing c++ guard)\n" ;; - 2*) printf " FAIL (cannot compile)\n" ;; - 3*) printf " FAIL (missing include of sdkconfig.h)\n" ;; - esac - if [[ ! "$ret" == 0 ]]; then - echo ${rel_inc_file} >> ${list_of_failed_includes} - fi - fi - done - done < $WORK_ITEM_PIPE; -} - -# check_include_dirs -# -# Check all header files in directories listed in include_dirs variable. -# Places the list of header files where suspects missing C++ guard to file. -# Places the list of header files where compilation issues found to file. -# Calls check_one_header function to check a single header file. -function check_include_dirs -{ - local include_dirs="$1" - local list_of_includes="$2" - local list_of_failed_includes="$3" - local include_dir_maxdepth=${INCLUDE_DIR_MAXDEPTH:-} - local grep_expr=$(mktemp) - - rm -f ${list_of_includes} ${list_of_failed_includes} - touch ${list_of_includes} ${list_of_failed_includes} - - if [ -z "${PROCESS_ALL_INCLUDES:-}" ]; then - # generate grep expr: 1) strip comments, 2) generate '\|\|...' - exclusions=`cat $EXCLUSIONS_LIST | sed '/^#/d'` - echo $exclusions | sed 's/ /\\|/g' > ${grep_expr} - fi; - - for include_dir in ${include_dirs}; do - echo "Processing include directory: ${include_dir}" - include_files=$(find ${include_dir} ${include_dir_maxdepth} -name '*.h') - for include_file in ${include_files}; do - if [ -z "${PROCESS_ALL_INCLUDES:-}" ] && echo "$include_file" | grep -q -f ${grep_expr}; then - continue - fi - echo ${include_file} >> ${list_of_includes} - done - done - rm -f ${grep_expr} -} - -# filter_exclusions -# -# Takes the list of files from stdin, and removes all files which -# are listed in file. -# Saves the final (filtered) list in file. -# The exclusions list file may contain comments and empty lines. -function filter_exclusions -{ - local exclusions_list_file=$1 - local filtered_list_file=$2 - local grep_expr=$(mktemp) - - # generate grep expr: 1) strip comments, 2) generate '\|\|...' - exclusions=`cat $exclusions_list_file | sed '/^#/d'` - echo $exclusions | sed 's/ /\\|/g' > ${grep_expr} - - # pass the expression as file (input goes from stdin) - grep -vf ${grep_expr} > ${filtered_list_file} || true - rm -f ${grep_expr} -} - -##################### main logic starts here ##################### - -EXCLUSIONS_LIST=${IDF_PATH}/tools/ci/check_public_headers_exceptions.txt -FAILED_INCLUDE_LIST=failed.txt -TOTAL_INCLUDE_LIST=include_list.txt -FILTERED_LIST=filtered.txt -LINE=--------------------------------------------------------------- -CC=${CC:-gcc} -CXX=${CXX:-g++} -TEST_PROJECT_DIR=${IDF_PATH}/examples/get-started/blink -MULTI_JOBS="1" -# jobs, load balancing vars, temp outputs -JOBS_LOAD_CHAR_FROM=64 -JOBS_LOAD_CHAR_TO=90 -let JOBS_LOAD_CHAR_SIZE=$JOBS_LOAD_CHAR_TO-$JOBS_LOAD_CHAR_FROM+1 -TEMP_OUT_FILE=/tmp/proc_out.txt. -WORK_ITEM_PIPE=/tmp/work_item_pipe - -# Process command line arguments -for i in $@; do - if [[ "$i" =~ "--verbose" ]]; then - echo "$0: Enabling verbose output" - VERBOSE="1" - fi - if [[ "$i" =~ "--process-all-files" ]]; then - echo "$0: Processing all files" - PROCESS_ALL_INCLUDES="1" - fi - if [[ "$i" =~ --include-maxdepth=* ]]; then - INCLUDE_DIR_MAXDEPTH="-maxdepth ${i##--include-maxdepth=}" - echo "$0: Searching for includes with: $INCLUDE_DIR_MAXDEPTH" - fi - if [[ "$i" =~ --jobs=* ]]; then - MULTI_JOBS="${i##--jobs=}" - echo "$0: Checking headers within multiple jobs: $MULTI_JOBS" - fi -done - -[[ -p $WORK_ITEM_PIPE ]] || mkfifo $WORK_ITEM_PIPE - -echo $LINE -# Get include directories -get_include_dirs_list ${TEST_PROJECT_DIR} -# Generate the main file to compiled while preincluding a header under test -create_test_main ${TEST_PROJECT_DIR}/build -# Get the list of include files to be processed -check_include_dirs "${INCLUDE_DIRS}" ${TOTAL_INCLUDE_LIST} ${FAILED_INCLUDE_LIST} - -# Launching separate processes (if $MULTI_JOBS defined) to check includes -# -# poor man's approach to load balancing: -# 1) sending work items (chars A-Z for atomicity) to a named pipe -# 2) spawning nr of processes to pick the work items from the pipe -# 3) every process reads one char at a time to analyze a batch of includes (batch=total_includes/nr_of_work_items) -# 4) waiting till all processes finish - -# running the jobs and marking the pid -for ((i=1; i<${MULTI_JOBS}; i++)); do - check_headers_in_process $i ${TOTAL_INCLUDE_LIST} ${FAILED_INCLUDE_LIST} > ${TEMP_OUT_FILE}${i} & - pids[${i}]=$! -done -# one job outputs directly to stdout (to show the script is alive) -check_headers_in_process 0 ${TOTAL_INCLUDE_LIST} ${FAILED_INCLUDE_LIST} & -pid0=$! - -# sending sequence of work items -for i in `seq $JOBS_LOAD_CHAR_FROM $JOBS_LOAD_CHAR_TO`; do - printf $(chr $i ) -done > $WORK_ITEM_PIPE - -# waiting for the processes to finish and print their outputs -for ((i=1; i<${MULTI_JOBS}; i++)); do - log_details "\nwaiting for process nr: $i\n" - wait ${pids[$i]} || true - cat ${TEMP_OUT_FILE}${i} - rm -f ${TEMP_OUT_FILE}${i} -done -# and the last one -wait $pid0 || true -rm -f $WORK_ITEM_PIPE - -# Filter out the known issues and report results -cat ${FAILED_INCLUDE_LIST} | filter_exclusions ${EXCLUSIONS_LIST} ${FILTERED_LIST} -echo $LINE - -if [ -s ${FILTERED_LIST} ]; then - echo "The following files failed:" - echo "" - cat ${FILTERED_LIST} - exit 1 -fi - -echo "No errors found" diff --git a/tools/ci/check_public_headers_exceptions.txt b/tools/ci/check_public_headers_exceptions.txt index 0ff628ab3e..dd6c0a951f 100644 --- a/tools/ci/check_public_headers_exceptions.txt +++ b/tools/ci/check_public_headers_exceptions.txt @@ -1,23 +1,23 @@ ### General ignore list # -components/xtensa/include/xtensa/* -components/xtensa/include/* -components/xtensa/esp32/include/xtensa/config/* +components/xtensa/include/xtensa/ +components/xtensa/include/ +components/xtensa/esp32/include/xtensa/config/ -components/newlib/platform_include/* +components/newlib/platform_include/ -components/freertos/include/freertos/* -components/freertos/xtensa/include/freertos/* +components/freertos/include/freertos/ +components/freertos/xtensa/include/freertos/ components/log/include/esp_log_internal.h -components/soc/include/hal/* -components/soc/include/soc/* +components/soc/include/hal/ +components/soc/include/soc/ components/esp_rom/include/esp32s2/rom/rsa_pss.h -components/esp_common/include/esp_private/* +components/esp_common/include/esp_private/ components/esp32/include/esp32/brownout.h components/esp32/include/esp32/spiram.h @@ -46,28 +46,28 @@ components/spi_flash/include/spi_flash_chip_generic.h components/bootloader_support/include/esp_app_format.h -components/wpa_supplicant/include/* -components/wpa_supplicant/port/* +components/wpa_supplicant/include/ +components/wpa_supplicant/port/ -components/mbedtls/port/include/* -components/mbedtls/mbedtls/include/mbedtls/* +components/mbedtls/port/include/ +components/mbedtls/mbedtls/include/mbedtls/ components/espcoredump/include/esp_core_dump.h -components/coap/* -components/nghttp/* -components/cbor/* +components/coap/ +components/nghttp/ +components/cbor/ -components/esp-tls/private_include/* +components/esp-tls/private_include/ -components/protobuf-c/* +components/protobuf-c/ components/mdns/include/mdns_console.h components/esp_serial_slave_link/include/esp_serial_slave_link/essl.h components/esp_serial_slave_link/include/esp_serial_slave_link/essl_sdio.h -components/expat/* +components/expat/ components/fatfs/vfs/vfs_fat_internal.h components/fatfs/src/ffconf.h @@ -78,9 +78,9 @@ components/freemodbus/common/include/esp_modbus_slave.h components/idf_test/include/idf_performance.h -components/json/cJSON/* +components/json/cJSON/ -components/libsodium/* +components/libsodium/ components/spiffs/include/spiffs_config.h @@ -100,3 +100,53 @@ components/heap/include/esp_heap_task_info.h components/esp_wifi/include/esp_private/wifi_os_adapter.h components/asio/port/include/esp_exception.h components/esp_common/include/esp_compiler.h + +### To be fixed: headers that rely on implicit inclusion +# +components/lwip/lwip/src/include/lwip/prot/nd6.h +components/lwip/port/esp32/include/netif/dhcp_state.h +components/soc/src/esp32/rtc_clk_common.h +components/soc/src/esp32/i2c_rtc_clk.h +components/soc/src/esp32/include/hal/cpu_ll.h +components/soc/src/esp32/include/hal/timer_ll.h +components/soc/src/esp32/include/hal/cpu_ll.h +components/soc/src/esp32/include/hal/timer_ll.h +components/esp_rom/include/esp32/rom/sha.h +components/esp_rom/include/esp32/rom/secure_boot.h +components/esp_rom/include/esp32s2/rom/spi_flash.h +components/esp_rom/include/esp32s2/rom/cache.h +components/esp_rom/include/esp32s2/rom/secure_boot.h +components/esp_rom/include/esp32s2/rom/opi_flash.h +components/esp_rom/include/esp32s2/rom/efuse.h +components/esp_common/include/esp_freertos_hooks.h +components/esp32/include/esp32/dport_access.h +components/esp32/include/rom/sha.h +components/esp32/include/rom/secure_boot.h +components/driver/esp32/include/touch_sensor.h +components/esp_ringbuf/include/freertos/ringbuf.h +components/efuse/esp32/include/esp_efuse_table.h +components/esp_wifi/include/esp_wifi_crypto_types.h +components/esp_wifi/include/esp_coexist_internal.h +components/esp_wifi/include/esp_wifi_netif.h +components/esp_wifi/include/smartconfig_ack.h +components/esp_wifi/include/esp_wifi_default.h +components/esp_wifi/include/esp_coexist_adapter.h +components/esp_event/include/esp_event_base.h +components/esp_netif/include/esp_netif_sta_list.h +components/esp_netif/include/esp_netif_defaults.h +components/esp_netif/include/esp_netif_net_stack.h +components/esp_netif/include/esp_netif_types.h +components/esp_netif/include/esp_netif_ip_addr.h +components/esp_netif/include/esp_netif_ppp.h +components/tcpip_adapter/include/tcpip_adapter_compatible/tcpip_adapter_compat.h +components/bootloader_support/include/bootloader_util.h +components/tcpip_adapter/include/tcpip_adapter_types.h +components/console/linenoise/linenoise.h +components/protocomm/include/transports/protocomm_httpd.h +components/fatfs/src/diskio.h +components/fatfs/diskio/diskio_sdmmc.h +components/openssl/include/openssl/ssl.h +components/ulp/include/ulp_common.h +components/lwip/include/apps/sntp/sntp.h +components/mbedtls/esp_crt_bundle/include/esp_crt_bundle.h +components/wifi_provisioning/include/wifi_provisioning/scheme_softap.h diff --git a/tools/ci/config/pre_check.yml b/tools/ci/config/pre_check.yml index 5fb9453328..4512a171cc 100644 --- a/tools/ci/config/pre_check.yml +++ b/tools/ci/config/pre_check.yml @@ -187,5 +187,5 @@ check_public_headers: - $BOT_LABEL_BUILD - $BOT_LABEL_REGULAR_TEST script: - - CC=xtensa-esp32-elf-gcc CXX=xtensa-esp32-elf-g++ tools/ci/check_public_headers.sh --verbose --process-all-files --jobs=4 + - python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32-elf-