#!/usr/bin/env python # # Command line tool to take in ESP-IDF sdkconfig files with project # settings and output data in multiple formats (update config, generate # header file, generate .cmake include file, documentation, etc). # # Used internally by the ESP-IDF build system. But designed to be # non-IDF-specific. # # SPDX-FileCopyrightText: 2018-2021 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import argparse import json import os import os.path import re import sys import tempfile import textwrap from collections import defaultdict import gen_kconfig_doc import kconfiglib __version__ = '0.1' class DeprecatedOptions(object): _REN_FILE = 'sdkconfig.rename' _DEP_OP_BEGIN = '# Deprecated options for backward compatibility' _DEP_OP_END = '# End of deprecated options' _RE_DEP_OP_BEGIN = re.compile(_DEP_OP_BEGIN) _RE_DEP_OP_END = re.compile(_DEP_OP_END) def __init__(self, config_prefix, path_rename_files=[]): self.config_prefix = config_prefix # r_dic maps deprecated options to new options; rev_r_dic maps in the opposite direction self.r_dic, self.rev_r_dic = self._parse_replacements(path_rename_files) # note the '=' at the end of regex for not getting partial match of configs. # Also match if the config option is followed by a whitespace, this is the case # in sdkconfig.defaults files contaning "# CONFIG_MMM_NNN is not set". self._RE_CONFIG = re.compile(r'{}(\w+)(=|\s+)'.format(self.config_prefix)) def _parse_replacements(self, repl_paths): rep_dic = {} rev_rep_dic = defaultdict(list) def remove_config_prefix(string): if string.startswith(self.config_prefix): return string[len(self.config_prefix):] raise RuntimeError('Error in {} (line {}): Config {} is not prefixed with {}' ''.format(rep_path, line_number, string, self.config_prefix)) for rep_path in repl_paths: with open(rep_path) as f_rep: for line_number, line in enumerate(f_rep, start=1): sp_line = line.split() if len(sp_line) == 0 or sp_line[0].startswith('#'): # empty line or comment continue if len(sp_line) != 2 or not all(x.startswith(self.config_prefix) for x in sp_line): raise RuntimeError('Syntax error in {} (line {})'.format(rep_path, line_number)) if sp_line[0] in rep_dic: raise RuntimeError('Error in {} (line {}): Replacement {} exist for {} and new ' 'replacement {} is defined'.format(rep_path, line_number, rep_dic[sp_line[0]], sp_line[0], sp_line[1])) (dep_opt, new_opt) = (remove_config_prefix(x) for x in sp_line) rep_dic[dep_opt] = new_opt rev_rep_dic[new_opt].append(dep_opt) return rep_dic, rev_rep_dic def get_deprecated_option(self, new_option): return self.rev_r_dic.get(new_option, []) def get_new_option(self, deprecated_option): return self.r_dic.get(deprecated_option, None) def replace(self, sdkconfig_in, sdkconfig_out): replace_enabled = True with open(sdkconfig_in, 'r') as f_in, open(sdkconfig_out, 'w') as f_out: for line_num, line in enumerate(f_in, start=1): if self._RE_DEP_OP_BEGIN.search(line): replace_enabled = False elif self._RE_DEP_OP_END.search(line): replace_enabled = True elif replace_enabled: m = self._RE_CONFIG.search(line) if m and m.group(1) in self.r_dic: depr_opt = self.config_prefix + m.group(1) new_opt = self.config_prefix + self.r_dic[m.group(1)] line = line.replace(depr_opt, new_opt) print('{}:{} {} was replaced with {}'.format(sdkconfig_in, line_num, depr_opt, new_opt)) f_out.write(line) def append_doc(self, config, visibility, path_output): def option_was_written(opt): # named choices were written if any of the symbols in the choice were visible if new_opt in config.named_choices: syms = config.named_choices[new_opt].syms for s in syms: if any(visibility.visible(node) for node in s.nodes): return True return False else: try: # otherwise if any of the nodes associated with the option was visible return any(visibility.visible(node) for node in config.syms[opt].nodes) except KeyError: return False if len(self.r_dic) > 0: with open(path_output, 'a') as f_o: header = 'Deprecated options and their replacements' f_o.write('.. _configuration-deprecated-options:\n\n{}\n{}\n\n'.format(header, '-' * len(header))) for dep_opt in sorted(self.r_dic): new_opt = self.r_dic[dep_opt] if option_was_written(new_opt) and (new_opt not in config.syms or config.syms[new_opt].choice is None): # everything except config for a choice (no link reference for those in the docs) f_o.write('- {}{} (:ref:`{}{}`)\n'.format(config.config_prefix, dep_opt, config.config_prefix, new_opt)) if new_opt in config.named_choices: # here are printed config options which were filtered out syms = config.named_choices[new_opt].syms for sym in syms: if sym.name in self.rev_r_dic: # only if the symbol has been renamed dep_names = self.rev_r_dic[sym.name] dep_names = [config.config_prefix + name for name in dep_names] # config options doesn't have references f_o.write(' - {}\n'.format(', '.join(dep_names))) def append_config(self, config, path_output): tmp_list = [] def append_config_node_process(node): item = node.item if isinstance(item, kconfiglib.Symbol) and item.env_var is None: if item.name in self.rev_r_dic: c_string = item.config_string if c_string: for dep_name in self.rev_r_dic[item.name]: tmp_list.append(c_string.replace(self.config_prefix + item.name, self.config_prefix + dep_name)) for n in config.node_iter(): append_config_node_process(n) if len(tmp_list) > 0: with open(path_output, 'a') as f_o: f_o.write('\n{}\n'.format(self._DEP_OP_BEGIN)) f_o.writelines(tmp_list) f_o.write('{}\n'.format(self._DEP_OP_END)) def append_header(self, config, path_output): def _opt_defined(opt): if opt.orig_type in (kconfiglib.BOOL, kconfiglib.TRISTATE) and opt.str_value != 'n': opt_defined = True elif opt.orig_type in (kconfiglib.INT, kconfiglib.STRING, kconfiglib.HEX) and opt.str_value != '': opt_defined = True else: opt_defined = False return opt_defined if len(self.r_dic) > 0: with open(path_output, 'a') as f_o: f_o.write('\n/* List of deprecated options */\n') for dep_opt in sorted(self.r_dic): new_opt = self.r_dic[dep_opt] if new_opt in config.syms and _opt_defined(config.syms[new_opt]): f_o.write('#define {}{} {}{}\n'.format(self.config_prefix, dep_opt, self.config_prefix, new_opt)) def main(): parser = argparse.ArgumentParser(description='confgen.py v%s - Config Generation Tool' % __version__, prog=os.path.basename(sys.argv[0])) parser.add_argument('--config', help='Project configuration settings', nargs='?', default=None) parser.add_argument('--defaults', help='Optional project defaults file, used if --config file doesn\'t exist. ' 'Multiple files can be specified using multiple --defaults arguments.', nargs='?', default=[], action='append') parser.add_argument('--kconfig', help='KConfig file with config item definitions', required=True) parser.add_argument('--sdkconfig-rename', help='File with deprecated Kconfig options', required=False) parser.add_argument('--dont-write-deprecated', help='Do not write compatibility statements for deprecated values', action='store_true') parser.add_argument('--output', nargs=2, action='append', help='Write output file (format and output filename)', metavar=('FORMAT', 'FILENAME'), default=[]) parser.add_argument('--env', action='append', default=[], help='Environment to set when evaluating the config file', metavar='NAME=VAL') parser.add_argument('--env-file', type=argparse.FileType('r'), help='Optional file to load environment variables from. Contents ' 'should be a JSON object where each key/value pair is a variable.') parser.add_argument('--list-separator', choices=['space', 'semicolon'], default='space', help='Separator used in environment list variables (COMPONENT_SDKCONFIG_RENAMES)') args = parser.parse_args() for fmt, filename in args.output: if fmt not in OUTPUT_FORMATS.keys(): print("Format '%s' not recognised. Known formats: %s" % (fmt, OUTPUT_FORMATS.keys())) sys.exit(1) try: args.env = [(name,value) for (name,value) in (e.split('=',1) for e in args.env)] except ValueError: print("--env arguments must each contain =. To unset an environment variable, use 'ENV='") sys.exit(1) for name, value in args.env: os.environ[name] = value if args.env_file is not None: env = json.load(args.env_file) os.environ.update(env) config = kconfiglib.Kconfig(args.kconfig) config.warn_assign_redun = False config.warn_assign_override = False sdkconfig_renames_sep = ';' if args.list_separator == 'semicolon' else ' ' sdkconfig_renames = [args.sdkconfig_rename] if args.sdkconfig_rename else [] sdkconfig_renames_from_env = os.environ.get('COMPONENT_SDKCONFIG_RENAMES') if sdkconfig_renames_from_env: sdkconfig_renames += sdkconfig_renames_from_env.split(sdkconfig_renames_sep) deprecated_options = DeprecatedOptions(config.config_prefix, path_rename_files=sdkconfig_renames) if len(args.defaults) > 0: def _replace_empty_assignments(path_in, path_out): with open(path_in, 'r') as f_in, open(path_out, 'w') as f_out: for line_num, line in enumerate(f_in, start=1): line = line.strip() if line.endswith('='): line += 'n' print('{}:{} line was updated to {}'.format(path_out, line_num, line)) f_out.write(line) f_out.write('\n') # always load defaults first, so any items which are not defined in that config # will have the default defined in the defaults file for name in args.defaults: print('Loading defaults file %s...' % name) if not os.path.exists(name): raise RuntimeError('Defaults file not found: %s' % name) try: with tempfile.NamedTemporaryFile(prefix='confgen_tmp', delete=False) as f: temp_file1 = f.name with tempfile.NamedTemporaryFile(prefix='confgen_tmp', delete=False) as f: temp_file2 = f.name deprecated_options.replace(sdkconfig_in=name, sdkconfig_out=temp_file1) _replace_empty_assignments(temp_file1, temp_file2) config.load_config(temp_file2, replace=False) finally: try: os.remove(temp_file1) os.remove(temp_file2) except OSError: pass # If config file previously exists, load it if args.config and os.path.exists(args.config): # ... but replace deprecated options before that with tempfile.NamedTemporaryFile(prefix='confgen_tmp', delete=False) as f: temp_file = f.name try: deprecated_options.replace(sdkconfig_in=args.config, sdkconfig_out=temp_file) config.load_config(temp_file, replace=False) update_if_changed(temp_file, args.config) finally: try: os.remove(temp_file) except OSError: pass if args.dont_write_deprecated: # The deprecated object was useful until now for replacements. Now it will be redefined with no configurations # and as the consequence, it won't generate output with deprecated statements. deprecated_options = DeprecatedOptions('', path_rename_files=[]) # Output the files specified in the arguments for output_type, filename in args.output: with tempfile.NamedTemporaryFile(prefix='confgen_tmp', delete=False) as f: temp_file = f.name try: output_function = OUTPUT_FORMATS[output_type] output_function(deprecated_options, config, temp_file) update_if_changed(temp_file, filename) finally: try: os.remove(temp_file) except OSError: pass def write_config(deprecated_options, config, filename): CONFIG_HEADING = """# # Automatically generated file. DO NOT EDIT. # Espressif IoT Development Framework (ESP-IDF) Project Configuration # """ config.write_config(filename, header=CONFIG_HEADING) deprecated_options.append_config(config, filename) def write_min_config(deprecated_options, config, filename): target_symbol = config.syms['IDF_TARGET'] # 'esp32` is harcoded here because the default value of IDF_TARGET is set on the first run from the environment # variable. I.E. `esp32 is not defined as default value. write_target = target_symbol.str_value != 'esp32' CONFIG_HEADING = textwrap.dedent('''\ # This file was generated using idf.py save-defconfig. It can be edited manually. # Espressif IoT Development Framework (ESP-IDF) Project Minimal Configuration # {}\ '''.format(target_symbol.config_string if write_target else '')) config.write_min_config(filename, header=CONFIG_HEADING) def write_header(deprecated_options, config, filename): CONFIG_HEADING = """/* * Automatically generated file. DO NOT EDIT. * Espressif IoT Development Framework (ESP-IDF) Configuration Header */ #pragma once """ config.write_autoconf(filename, header=CONFIG_HEADING) deprecated_options.append_header(config, filename) def write_cmake(deprecated_options, config, filename): with open(filename, 'w') as f: tmp_dep_list = [] write = f.write prefix = config.config_prefix write("""# # Automatically generated file. DO NOT EDIT. # Espressif IoT Development Framework (ESP-IDF) Configuration cmake include file # """) configs_list = list() def write_node(node): sym = node.item if not isinstance(sym, kconfiglib.Symbol): return if sym.config_string: val = sym.str_value if sym.orig_type in (kconfiglib.BOOL, kconfiglib.TRISTATE) and val == 'n': val = '' # write unset values as empty variables elif sym.orig_type == kconfiglib.STRING: val = kconfiglib.escape(val) elif sym.orig_type == kconfiglib.HEX: val = hex(int(val, 16)) # ensure 0x prefix write('set({}{} "{}")\n'.format(prefix, sym.name, val)) configs_list.append(prefix + sym.name) dep_opts = deprecated_options.get_deprecated_option(sym.name) for opt in dep_opts: tmp_dep_list.append('set({}{} "{}")\n'.format(prefix, opt, val)) configs_list.append(prefix + opt) for n in config.node_iter(): write_node(n) write('set(CONFIGS_LIST {})'.format(';'.join(configs_list))) if len(tmp_dep_list) > 0: write('\n# List of deprecated options for backward compatibility\n') f.writelines(tmp_dep_list) def get_json_values(config): config_dict = {} def write_node(node): sym = node.item if not isinstance(sym, kconfiglib.Symbol): return if sym.config_string: val = sym.str_value if sym.type in [kconfiglib.BOOL, kconfiglib.TRISTATE]: val = (val != 'n') elif sym.type == kconfiglib.HEX: val = int(val, 16) elif sym.type == kconfiglib.INT: val = int(val) config_dict[sym.name] = val for n in config.node_iter(False): write_node(n) return config_dict def write_json(deprecated_options, config, filename): config_dict = get_json_values(config) with open(filename, 'w') as f: json.dump(config_dict, f, indent=4, sort_keys=True) def get_menu_node_id(node): """ Given a menu node, return a unique id which can be used to identify it in the menu structure Will either be the config symbol name, or a menu identifier 'slug' """ try: if not isinstance(node.item, kconfiglib.Choice): return node.item.name except AttributeError: pass result = [] while node.parent is not None: slug = re.sub(r'\W+', '-', node.prompt[0]).lower() result.append(slug) node = node.parent result = '-'.join(reversed(result)) return result def write_json_menus(deprecated_options, config, filename): existing_ids = set() result = [] # root level items node_lookup = {} # lookup from MenuNode to an item in result def write_node(node): try: json_parent = node_lookup[node.parent]['children'] except KeyError: assert node.parent not in node_lookup # if fails, we have a parent node with no "children" entity (ie a bug) json_parent = result # root level node # node.kconfig.y means node has no dependency, if node.dep is node.kconfig.y: depends = None else: depends = kconfiglib.expr_str(node.dep) try: # node.is_menuconfig is True in newer kconfiglibs for menus and choices as well is_menuconfig = node.is_menuconfig and isinstance(node.item, kconfiglib.Symbol) except AttributeError: is_menuconfig = False new_json = None if node.item == kconfiglib.MENU or is_menuconfig: new_json = {'type': 'menu', 'title': node.prompt[0], 'depends_on': depends, 'children': [], } if is_menuconfig: sym = node.item new_json['name'] = sym.name new_json['help'] = node.help new_json['is_menuconfig'] = is_menuconfig greatest_range = None if len(sym.ranges) > 0: # Note: Evaluating the condition using kconfiglib's expr_value # should have one condition which is true for min_range, max_range, cond_expr in sym.ranges: if kconfiglib.expr_value(cond_expr): greatest_range = [min_range, max_range] new_json['range'] = greatest_range elif isinstance(node.item, kconfiglib.Symbol): sym = node.item greatest_range = None if len(sym.ranges) > 0: # Note: Evaluating the condition using kconfiglib's expr_value # should have one condition which is true for min_range, max_range, cond_expr in sym.ranges: if kconfiglib.expr_value(cond_expr): base = 16 if sym.type == kconfiglib.HEX else 10 greatest_range = [int(min_range.str_value, base), int(max_range.str_value, base)] break new_json = { 'type': kconfiglib.TYPE_TO_STR[sym.type], 'name': sym.name, 'title': node.prompt[0] if node.prompt else None, 'depends_on': depends, 'help': node.help, 'range': greatest_range, 'children': [], } elif isinstance(node.item, kconfiglib.Choice): choice = node.item new_json = { 'type': 'choice', 'title': node.prompt[0], 'name': choice.name, 'depends_on': depends, 'help': node.help, 'children': [] } if new_json: node_id = get_menu_node_id(node) if node_id in existing_ids: raise RuntimeError('Config file contains two items with the same id: %s (%s). ' + 'Please rename one of these items to avoid ambiguity.' % (node_id, node.prompt[0])) new_json['id'] = node_id json_parent.append(new_json) node_lookup[node] = new_json for n in config.node_iter(): write_node(n) with open(filename, 'w') as f: f.write(json.dumps(result, sort_keys=True, indent=4)) def write_docs(deprecated_options, config, filename): try: target = os.environ['IDF_TARGET'] except KeyError: print('IDF_TARGET environment variable must be defined!') sys.exit(1) visibility = gen_kconfig_doc.ConfigTargetVisibility(config, target) gen_kconfig_doc.write_docs(config, visibility, filename) deprecated_options.append_doc(config, visibility, filename) def update_if_changed(source, destination): with open(source, 'r') as f: source_contents = f.read() if os.path.exists(destination): with open(destination, 'r') as f: dest_contents = f.read() if source_contents == dest_contents: return # nothing to update with open(destination, 'w') as f: f.write(source_contents) OUTPUT_FORMATS = {'config': write_config, 'header': write_header, 'cmake': write_cmake, 'docs': write_docs, 'json': write_json, 'json_menus': write_json_menus, 'savedefconfig': write_min_config, } class FatalError(RuntimeError): """ Class for runtime errors (not caused by bugs but by user input). """ pass if __name__ == '__main__': try: main() except FatalError as e: print('A fatal error occurred: %s' % e) sys.exit(2)