#!/usr/bin/env python # coding=utf-8 # # SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD # # SPDX-License-Identifier: Apache-2.0 # # This script converts space-separated EXTRA_COMPONENT_DIRS and COMPONENT_DIRS # CMake variables into semicolon-separated lists. # # IDF versions <=v4.3 didn't support spaces in paths to ESP-IDF or projects. # Therefore it was okay to use spaces as separators in EXTRA_COMPONENT_DIRS, # same as it was done in the legacy GNU Make based build system. # CMake build system used 'spaces2list' function to convert space-separated # variables into semicolon-separated lists, replacing every space with a # semicolon. # # In IDF 4.4 and later, spaces in project path and ESP-IDF path are supported. # This means that EXTRA_COMPONENT_DIRS and COMPONENT_DIRS variables now should # be semicolon-separated CMake lists. # # To provide compatibility with the projects written for older ESP-IDF versions, # this script attempts to convert these space-separated variables into semicolon- # separated ones. Note that in general this cannot be done unambiguously, so this # script will still report an error if there are multiple ways to interpret the # variable, and ask the user to fix the project CMakeLists.txt file. # import argparse import os import pprint import sys import textwrap import typing import unittest class PathSplitError(RuntimeError): pass def main() -> None: parser = argparse.ArgumentParser() parser.add_argument('--var-name', required=True, help='Name of CMake variable, for printing errors and warnings') parser.add_argument('in_variable', help='Input variable, may contain a mix of spaces and semicolons as separators') args = parser.parse_args() # Initially split the paths by semicolons semicolon_separated_parts = args.in_variable.split(';') # Every resulting part may contain space separators. Handle each part: paths = [] ctx = dict(warnings=False) errors = False for part in semicolon_separated_parts: def warning_cb(warning_str: str) -> None: print('\n '.join( textwrap.wrap('Warning: in CMake variable {}: {}'.format(args.var_name, warning_str), width=120, break_on_hyphens=False)), file=sys.stderr) ctx['warnings'] = True try: paths += split_paths_by_spaces(part, warning_cb=warning_cb) except PathSplitError as e: print('\n '.join(textwrap.wrap('Error: in CMake variable {}: {}'.format(args.var_name, str(e)), width=120, break_on_hyphens=False)), file=sys.stderr) errors = True if errors or ctx['warnings']: print(textwrap.dedent(""" Note: In ESP-IDF v4.4 and later, COMPONENT_DIRS and EXTRA_COMPONENT_DIRS should be defined as CMake lists, not as space separated strings. Examples: * set(EXTRA_COMPONENT_DIRS path/to/components path/to/more/components) # Correct, EXTRA_COMPONENT_DIRS is defined as a CMake list, with two paths added * list(APPEND EXTRA_COMPONENT_DIRS path/to/component) list(APPEND EXTRA_COMPONENT_DIRS path/to/more/components) # Correct, use when building EXTRA_COMPONENT_DIRS incrementally * set(EXTRA_COMPONENT_DIRS path/to/components "another/path with space/components") # Literal path with spaces has to be quoted * set(EXTRA_COMPONENT_DIRS $ENV{MY_PATH}/components dir/more_components) # Correct, even if MY_PATH contains spaces * set(EXTRA_COMPONENT_DIRS ${ROOT}/component1 ${ROOT}/component2 ${ROOT}/component3) # Correct, even if ROOT contains spaces Avoid string concatenation! set(EXTRA_COMPONENT_DIRS "${EXTRA_COMPONENT_DIRS} component1") set(EXTRA_COMPONENT_DIRS "${EXTRA_COMPONENT_DIRS} component2") # Incorrect. String "component1 component2" may indicate a single directory # name with a space, or two directory names separated by space. Instead use: list(APPEND component1) list(APPEND component2) Defining COMPONENT_DIRS and EXTRA_COMPONENT_DIRS as CMake lists is backwards compatible with ESP-IDF 4.3 and below. (If you think these variables are defined correctly in your project and this message is not relevant, please report this as an issue.) """), file=sys.stderr) print('Diagnostic info: {} was invoked in {} with arguments: {}'.format( sys.argv[0], os.getcwd(), sys.argv[1:] ), file=sys.stderr) if errors: raise SystemExit(1) sys.stdout.write(';'.join(paths)) sys.stdout.flush() def split_paths_by_spaces(src: str, path_exists_cb: typing.Callable[[str], bool] = os.path.exists, warning_cb: typing.Optional[typing.Callable[[str], None]] = None) -> typing.List[str]: if ' ' not in src: # no spaces, complete string should be the path return [src] def path_exists_or_empty(path: str) -> bool: return path == '' or path_exists_cb(path) # remove leading and trailing spaces delayed_warnings = [] trimmed = src.lstrip(' ') if trimmed != src: delayed_warnings.append("Path component '{}' contains leading spaces".format(src)) src = trimmed trimmed = src.rstrip(' ') if trimmed != src: delayed_warnings.append("Path component '{}' contains trailing spaces".format(src)) src = trimmed # Enumerate all possible ways to split the string src into paths by spaces. # The number of these ways is equal to sum(C(n, k), 0<=k 1: warning_cb("Path component '{}' contains a space separator. It was automatically split into {}".format( src, pprint.pformat(result) )) for w in delayed_warnings: warning_cb(w) return result if num_candidates == 0: raise PathSplitError(("Didn't find a valid way to split path '{}'. " 'This error may be reported if one or more paths ' "are separated with spaces, and at least one path doesn't exist.").format(src)) # if num_candidates > 1 raise PathSplitError("Found more than one valid way to split path '{}':{}".format( src, ''.join('\n\t- ' + pprint.pformat(p) for p in valid_ways_to_split) )) def selective_join(parts: typing.List[str], n: int) -> typing.List[str]: """ Given the list of N+1 strings, and an integer n in [0, 2**N - 1] range, concatenate i-th and (i+1)-th string with space inbetween if bit i is not set in n. Examples: selective_join(['a', 'b', 'c'], 0b00) == ['a b c'] selective_join(['a', 'b', 'c'], 0b01) == ['a', 'b c'] selective_join(['a', 'b', 'c'], 0b10) == ['a b', 'c'] selective_join(['a', 'b', 'c'], 0b11) == ['a', 'b', 'c'] This function is used as part of finding all the ways to split a string by spaces. :param parts: Strings to join :param n: Integer (bit map) to set the positions to join :return: resulting list of strings """ result = [] concatenated = [parts[0]] for part in parts[1:]: if n & 1: result.append(' '.join(concatenated)) concatenated = [part] else: concatenated.append(part) n >>= 1 if concatenated: result.append(' '.join(concatenated)) return result class HelperTests(unittest.TestCase): def test_selective_join(self) -> None: self.assertListEqual(['a b c'], selective_join(['a', 'b', 'c'], 0b00)) self.assertListEqual(['a', 'b c'], selective_join(['a', 'b', 'c'], 0b01)) self.assertListEqual(['a b', 'c'], selective_join(['a', 'b', 'c'], 0b10)) self.assertListEqual(['a', 'b', 'c'], selective_join(['a', 'b', 'c'], 0b11)) class SplitTests(unittest.TestCase): def test_split_paths_absolute(self) -> None: self.check_paths_concatenated('/absolute/path/one', '/absolute/path/two') def test_split_paths_absolute_spaces(self) -> None: self.check_paths_concatenated('/absolute/path with spaces') self.check_paths_concatenated('/absolute/path with more spaces') self.check_paths_concatenated('/absolute/path with spaces/one', '/absolute/path with spaces/two') self.check_paths_concatenated('/absolute/path with spaces/one', '/absolute/path with spaces/two', '/absolute/path with spaces/three') def test_split_paths_absolute_relative(self) -> None: self.check_paths_concatenated('/absolute/path/one', 'two') def test_split_paths_relative(self) -> None: self.check_paths_concatenated('one', 'two') def test_split_paths_absolute_spaces_relative(self) -> None: self.check_paths_concatenated('/absolute/path with spaces/one', 'two') def test_split_paths_ambiguous(self) -> None: self.check_paths_concatenated_ambiguous('/absolute/path one', 'two', additional_paths_exist=['/absolute/path', 'one']) self.check_paths_concatenated_ambiguous('/path ', '/path', additional_paths_exist=['/path /path']) def test_split_paths_nonexistent(self) -> None: self.check_paths_concatenated_nonexistent('one', 'two') def test_split_paths_extra_whitespace(self) -> None: paths = ['/path'] path_exists = self.path_exists_by_list(paths) self.assertListEqual(paths, split_paths_by_spaces(' /path', path_exists_cb=path_exists)) self.assertListEqual(paths, split_paths_by_spaces('/path ', path_exists_cb=path_exists)) self.assertListEqual(paths + paths, split_paths_by_spaces('/path /path', path_exists_cb=path_exists)) def test_split_paths_warnings(self) -> None: paths = ['/path'] ctx = {'warnings': []} # type: typing.Dict[str, typing.List[str]] def add_warning(warning: str) -> None: ctx['warnings'].append(warning) path_exists = self.path_exists_by_list(paths) self.assertListEqual(paths, split_paths_by_spaces(' /path', path_exists_cb=path_exists, warning_cb=add_warning)) self.assertEqual(1, len(ctx['warnings'])) self.assertIn('leading', ctx['warnings'][0]) ctx['warnings'] = [] self.assertListEqual(paths, split_paths_by_spaces('/path ', path_exists_cb=path_exists, warning_cb=add_warning)) self.assertEqual(1, len(ctx['warnings'])) self.assertIn('trailing', ctx['warnings'][0]) ctx['warnings'] = [] self.assertListEqual(paths + paths, split_paths_by_spaces('/path /path', path_exists_cb=path_exists, warning_cb=add_warning)) self.assertEqual(1, len(ctx['warnings'])) self.assertIn('contains a space separator', ctx['warnings'][0]) @staticmethod def path_exists_by_list(paths_which_exist: typing.List[str]) -> typing.Callable[[str], bool]: """ Returns a function to check whether a path exists, similar to os.path.exists, but instead of checking for files on the real filesystem it considers only the paths provided in 'paths_which_exist' argument. :param paths_which_exist: list of paths which should be considered as existing :return: function to check if path exists """ all_paths = set() for path in paths_which_exist or []: # for path /a/b/c, add it and also add components of the path: /a, /a/b end = len(path) while end > 0: all_paths.add(path[0:end]) end = path.rfind('/', 0, end) def path_exists(path: str) -> bool: return path in all_paths return path_exists def split_paths_concatenated_base(self, paths_to_concatentate: typing.List[str], paths_existing: typing.List[str]) -> typing.List[str]: concatenated = ' '.join(paths_to_concatentate) path_exists = self.path_exists_by_list(paths_existing) return split_paths_by_spaces(concatenated, path_exists_cb=path_exists) def check_paths_concatenated(self, *args: str) -> None: paths = [*args] paths_split = self.split_paths_concatenated_base(paths_to_concatentate=paths, paths_existing=paths) self.assertListEqual(paths, paths_split) def check_paths_concatenated_ambiguous(self, *args: str, additional_paths_exist: typing.Optional[typing.List[str]] = None) -> None: paths = [*args] self.assertRaises(PathSplitError, self.split_paths_concatenated_base, paths_to_concatentate=paths, paths_existing=paths + (additional_paths_exist or [])) def check_paths_concatenated_nonexistent(self, *args: str, additional_paths_exist: typing.List[str] = None) -> None: paths = [*args] self.assertRaises(PathSplitError, self.split_paths_concatenated_base, paths_to_concatentate=paths, paths_existing=additional_paths_exist) if __name__ == '__main__': main()