#!/usr/bin/env python # # Checks all public headers in IDF in the ci # # Copyright 2020 Espressif Systems (Shanghai) PTE LTD # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from __future__ import print_function, unicode_literals import argparse import fnmatch import json import os import queue import re import subprocess import tempfile from io import open from threading import Event, Thread class HeaderFailed(Exception): """Base header failure exeption""" pass class HeaderFailedSdkconfig(HeaderFailed): def __str__(self): return 'Sdkconfig Error' class HeaderFailedBuildError(HeaderFailed): def __str__(self): return 'Header Build Error' class HeaderFailedCppGuardMissing(HeaderFailed): def __str__(self): return 'Header Missing C++ Guard' class HeaderFailedContainsCode(HeaderFailed): def __str__(self): return 'Header Produced non-zero object' # Creates a temp file and returns both output as a string and a file name # def exec_cmd_to_temp_file(what, suffix=''): out_file = tempfile.NamedTemporaryFile(suffix=suffix, delete=False) rc, out, err = exec_cmd(what, out_file) with open(out_file.name, 'r', encoding='utf-8') as f: out = f.read() return rc, out, err, out_file.name def exec_cmd(what, out_file=subprocess.PIPE): p = subprocess.Popen(what, stdin=subprocess.PIPE, stdout=out_file, stderr=subprocess.PIPE) output, err = p.communicate() rc = p.returncode output = output.decode('utf-8') if output is not None else None err = err.decode('utf-8') if err is not None else None return rc, output, err class PublicHeaderChecker: # Intermediate results COMPILE_ERR_REF_CONFIG_HDR_FAILED = 1 # -> Cannot compile and failed with injected SDKCONFIG #error (header FAILs) COMPILE_ERR_ERROR_MACRO_HDR_OK = 2 # -> Cannot compile, but failed with "#error" directive (header seems OK) COMPILE_ERR_HDR_FAILED = 3 # -> Cannot compile with another issue, logged if verbose (header FAILs) PREPROC_OUT_ZERO_HDR_OK = 4 # -> Both preprocessors produce zero out (header file is OK) PREPROC_OUT_SAME_HRD_FAILED = 5 # -> Both preprocessors produce the same, non-zero output (header file FAILs) PREPROC_OUT_DIFFERENT_WITH_EXT_C_HDR_OK = 6 # -> Both preprocessors produce different, non-zero output with extern "C" (header seems OK) PREPROC_OUT_DIFFERENT_NO_EXT_C_HDR_FAILED = 7 # -> Both preprocessors produce different, non-zero output without extern "C" (header fails) def log(self, message, debug=False): if self.verbose or debug: print(message) def __init__(self, verbose=False, jobs=1, prefix=None): self.gcc = '{}gcc'.format(prefix) self.gpp = '{}g++'.format(prefix) self.verbose = verbose self.jobs = jobs self.prefix = prefix self.extern_c = re.compile(r'extern "C"') self.error_macro = re.compile(r'#error') self.error_orphan_kconfig = re.compile(r'#error CONFIG_VARS_USED_WHILE_SDKCONFIG_NOT_INCLUDED') self.kconfig_macro = re.compile(r'\bCONFIG_[A-Z0-9_]+') self.assembly_nocode = r'^\s*(\.file|\.text|\.ident).*$' self.check_threads = [] self.job_queue = queue.Queue() self.failed_queue = queue.Queue() self.terminate = Event() def __enter__(self): for i in range(self.jobs): t = Thread(target=self.check_headers, args=(i, )) self.check_threads.append(t) t.start() return self def __exit__(self, exc_type, exc_value, traceback): self.terminate.set() for t in self.check_threads: t.join() # thread function process incoming header file from a queue def check_headers(self, num): while not self.terminate.is_set(): if not self.job_queue.empty(): task = self.job_queue.get() if task is None: self.terminate.set() else: try: self.check_one_header(task, num) except HeaderFailed as e: self.failed_queue.put('{}: Failed! {}'.format(task, e)) except Exception as e: # Makes sure any unexpected exceptions causes the program to terminate self.failed_queue.put('{}: Failed! {}'.format(task, e)) self.terminate.set() raise def get_failed(self): return list(self.failed_queue.queue) def join(self): for t in self.check_threads: while t.is_alive() and not self.terminate.is_set(): t.join(1) # joins with timeout to respond to keyboard interrupt # Checks one header calling: # - preprocess_one_header() to test and compare preprocessor outputs # - check_no_code() to test if header contains some executable code # Procedure # 1) Preprocess the include file with C preprocessor and with CPP preprocessor # - Pass the test if the preprocessor outputs are the same and whitespaces only (#define only header) # - Fail the test if the preprocessor outputs are the same (but with some code) # - If outputs different, continue with 2) # 2) Strip out all include directives to generate "temp.h" # 3) Preprocess the temp.h the same way in (1) # - Pass the test if the preprocessor outputs are the same and whitespaces only (#include only header) # - Fail the test if the preprocessor outputs are the same (but with some code) # - If outputs different, pass the test # 4) If header passed the steps 1) and 3) test that it produced zero assembly code def check_one_header(self, header, num): res = self.preprocess_one_header(header, num) if res == self.COMPILE_ERR_REF_CONFIG_HDR_FAILED: raise HeaderFailedSdkconfig() elif res == self.COMPILE_ERR_ERROR_MACRO_HDR_OK: return self.compile_one_header(header) elif res == self.COMPILE_ERR_HDR_FAILED: raise HeaderFailedBuildError() elif res == self.PREPROC_OUT_ZERO_HDR_OK: return self.compile_one_header(header) elif res == self.PREPROC_OUT_SAME_HRD_FAILED: raise HeaderFailedCppGuardMissing() else: self.compile_one_header(header) temp_header = None try: _, _, _, temp_header = exec_cmd_to_temp_file(['sed', '/#include/d; /#error/d', header], suffix='.h') res = self.preprocess_one_header(temp_header, num, ignore_sdkconfig_issue=True) if res == self.PREPROC_OUT_SAME_HRD_FAILED: raise HeaderFailedCppGuardMissing() elif res == self.PREPROC_OUT_DIFFERENT_NO_EXT_C_HDR_FAILED: raise HeaderFailedCppGuardMissing() finally: if temp_header: os.unlink(temp_header) def compile_one_header(self, header): rc, out, err = exec_cmd([self.gcc, '-S', '-o-', '-include', header, self.main_c] + self.include_dir_flags) if rc == 0: if not re.sub(self.assembly_nocode, '', out, flags=re.M).isspace(): raise HeaderFailedContainsCode() return # Header OK: produced zero code self.log('{}: FAILED: compilation issue'.format(header), True) self.log(err, True) raise HeaderFailedBuildError() def preprocess_one_header(self, header, num, ignore_sdkconfig_issue=False): all_compilation_flags = ['-w', '-P', '-E', '-DESP_PLATFORM', '-include', header, self.main_c] + self.include_dir_flags if not ignore_sdkconfig_issue: # just strip commnets to check for CONFIG_... macros rc, out, err = exec_cmd([self.gcc, '-fpreprocessed', '-dD', '-P', '-E', header] + self.include_dir_flags) if re.search(self.kconfig_macro, out): # enable defined #error if sdkconfig.h not included all_compilation_flags.append('-DIDF_CHECK_SDKCONFIG_INCLUDED') try: # compile with C++, check for errors, outputs for a temp file rc, cpp_out, err, cpp_out_file = exec_cmd_to_temp_file([self.gpp, '--std=c++17'] + all_compilation_flags) if rc != 0: if re.search(self.error_macro, err): if re.search(self.error_orphan_kconfig, err): self.log('{}: CONFIG_VARS_USED_WHILE_SDKCONFIG_NOT_INCLUDED'.format(header), True) return self.COMPILE_ERR_REF_CONFIG_HDR_FAILED self.log('{}: Error directive failure: OK'.format(header)) return self.COMPILE_ERR_ERROR_MACRO_HDR_OK self.log('{}: FAILED: compilation issue'.format(header), True) self.log(err) return self.COMPILE_ERR_HDR_FAILED # compile with C compiler, outputs to another temp file rc, c99_out, err, c99_out_file = exec_cmd_to_temp_file([self.gcc, '--std=c99'] + all_compilation_flags) if rc != 0: self.log('{} FAILED should never happen'.format(header)) return self.COMPILE_ERR_HDR_FAILED # diff the two outputs rc, diff, err = exec_cmd(['diff', c99_out_file, cpp_out_file]) if not diff or diff.isspace(): if not cpp_out or cpp_out.isspace(): self.log('{} The same, but empty out - OK'.format(header)) return self.PREPROC_OUT_ZERO_HDR_OK self.log('{} FAILED C and C++ preprocessor output is the same!'.format(header), True) return self.PREPROC_OUT_SAME_HRD_FAILED if re.search(self.extern_c, diff): self.log('{} extern C present - OK'.format(header)) return self.PREPROC_OUT_DIFFERENT_WITH_EXT_C_HDR_OK self.log('{} Different but no extern C - FAILED'.format(header), True) return self.PREPROC_OUT_DIFFERENT_NO_EXT_C_HDR_FAILED finally: os.unlink(cpp_out_file) try: os.unlink(c99_out_file) except Exception: pass # Get compilation data from an example to list all public header files def list_public_headers(self, ignore_dirs, ignore_files, only_dir=None): idf_path = os.getenv('IDF_PATH') project_dir = os.path.join(idf_path, 'examples', 'get-started', 'blink') subprocess.check_call(['idf.py', 'reconfigure'], cwd=project_dir) build_commands_json = os.path.join(project_dir, 'build', 'compile_commands.json') with open(build_commands_json, 'r', encoding='utf-8') as f: build_command = json.load(f)[0]['command'].split() include_dir_flags = [] include_dirs = [] # process compilation flags (includes and defines) for item in build_command: if item.startswith('-I'): include_dir_flags.append(item) if 'components' in item: include_dirs.append(item[2:]) # Removing the leading "-I" if item.startswith('-D'): include_dir_flags.append(item.replace('\\','')) # removes escaped quotes, eg: -DMBEDTLS_CONFIG_FILE=\\\"mbedtls/esp_config.h\\\" include_dir_flags.append('-I' + os.path.join(project_dir, 'build', 'config')) include_dir_flags.append('-DCI_HEADER_CHECK') sdkconfig_h = os.path.join(project_dir, 'build', 'config', 'sdkconfig.h') # prepares a main_c file for easier sdkconfig checks and avoid compilers warning when compiling headers directly with open(sdkconfig_h, 'a') as f: f.write('#define IDF_SDKCONFIG_INCLUDED') main_c = os.path.join(project_dir, 'build', 'compile.c') with open(main_c, 'w') as f: f.write('#if defined(IDF_CHECK_SDKCONFIG_INCLUDED) && ! defined(IDF_SDKCONFIG_INCLUDED)\n' '#error CONFIG_VARS_USED_WHILE_SDKCONFIG_NOT_INCLUDED\n' '#endif') # processes public include dirs, removing ignored files all_include_files = [] files_to_check = [] for d in include_dirs: if only_dir is not None and not os.path.relpath(d, idf_path).startswith(only_dir): self.log('{} - directory ignored (not in "{}")'.format(d, only_dir)) continue if os.path.relpath(d, idf_path).startswith(tuple(ignore_dirs)): self.log('{} - directory ignored'.format(d)) continue for root, dirnames, filenames in os.walk(d): for filename in fnmatch.filter(filenames, '*.h'): all_include_files.append(os.path.join(root, filename)) self.main_c = main_c self.include_dir_flags = include_dir_flags ignore_files = set(ignore_files) # processes public include files, removing ignored files for f in all_include_files: rel_path_file = os.path.relpath(f, idf_path) if any([os.path.commonprefix([d, rel_path_file]) == d for d in ignore_dirs]): self.log('{} - file ignored (inside ignore dir)'.format(f)) continue if rel_path_file in ignore_files: self.log('{} - file ignored'.format(f)) continue files_to_check.append(f) # removes duplicates and places headers to a work queue for f in set(files_to_check): self.job_queue.put(f) self.job_queue.put(None) # to indicate the last job def check_all_headers(): parser = argparse.ArgumentParser('Public header checker file') parser.add_argument('--verbose', '-v', help='enables verbose mode', action='store_true') parser.add_argument('--jobs', '-j', help='number of jobs to run checker', default=1, type=int) parser.add_argument('--prefix', '-p', help='compiler prefix', default='xtensa-esp32-elf-', type=str) parser.add_argument('--exclude-file', '-e', help='exception file', default='check_public_headers_exceptions.txt', type=str) parser.add_argument('--only-dir', '-d', help='reduce the analysis to this directory only', default=None, type=str) args = parser.parse_args() # process excluded files and dirs exclude_file = os.path.join(os.path.dirname(__file__), args.exclude_file) with open(exclude_file, 'r', encoding='utf-8') as f: lines = [line.rstrip() for line in f] ignore_files = [] ignore_dirs = [] for line in lines: if not line or line.isspace() or line.startswith('#'): continue if os.path.isdir(line): ignore_dirs.append(line) else: ignore_files.append(line) # start header check with PublicHeaderChecker(args.verbose, args.jobs, args.prefix) as header_check: header_check.list_public_headers(ignore_dirs, ignore_files, only_dir=args.only_dir) try: header_check.join() failures = header_check.get_failed() if len(failures) > 0: for failed in failures: print(failed) exit(1) print('No errors found') except KeyboardInterrupt: print('Keyboard interrupt') if __name__ == '__main__': check_all_headers()