From 165c0d852bd38c0ff3196cdf77508391fc9e4d5c Mon Sep 17 00:00:00 2001 From: Marek Fiala Date: Mon, 30 Jan 2023 10:40:30 +0100 Subject: [PATCH] Tools: Rewrite build system unit tests to python - other target tests --- .../build_test/sdkconfig.ci.ethernet_disabled | 2 + tools/test_build_system/MIGRATION.md | 26 ++--- tools/test_build_system/conftest.py | 9 +- tools/test_build_system/test_build.py | 45 +++++--- .../test_build_system_helpers/__init__.py | 4 +- .../test_build_system_helpers/idf_utils.py | 17 ++- .../test_non_default_target.py | 104 ++++++++++++++++++ 7 files changed, 173 insertions(+), 34 deletions(-) create mode 100644 tools/test_apps/system/build_test/sdkconfig.ci.ethernet_disabled create mode 100644 tools/test_build_system/test_non_default_target.py diff --git a/tools/test_apps/system/build_test/sdkconfig.ci.ethernet_disabled b/tools/test_apps/system/build_test/sdkconfig.ci.ethernet_disabled new file mode 100644 index 0000000000..798bb92d5b --- /dev/null +++ b/tools/test_apps/system/build_test/sdkconfig.ci.ethernet_disabled @@ -0,0 +1,2 @@ +CONFIG_ETH_USE_SPI_ETHERNET=n +CONFIG_ETH_USE_ESP32_EMAC=n diff --git a/tools/test_build_system/MIGRATION.md b/tools/test_build_system/MIGRATION.md index 8d0d389d6f..26f3a8d45b 100644 --- a/tools/test_build_system/MIGRATION.md +++ b/tools/test_build_system/MIGRATION.md @@ -34,19 +34,19 @@ idf.py can build with Unix Makefiles | test_build.py::test_build_with_generator_ Can build with IDF_PATH set via cmake cache not environment | test_build.py::test_build_with_cmake_and_idf_path_unset | Can build with IDF_PATH unset and inferred by build system | test_build.py::test_build_with_cmake_and_idf_path_unset | Can build with IDF_PATH unset and inferred by cmake when Kconfig needs it to be set | test_build.py::test_build_with_cmake_and_idf_path_unset | -can build with phy_init_data | | -can build with ethernet component disabled | | -Compiler flags on build command line are taken into account | | -Compiler flags cannot be overwritten | | -Can override IDF_TARGET from environment | | -Can set target using idf.py -D | | -Can set target using -D as subcommand parameter for idf.py | | -Can set target using idf.py set-target | | -idf.py understands alternative target names | | -Can guess target from sdkconfig, if CMakeCache does not exist | | -Can set the default target using sdkconfig.defaults | | -IDF_TARGET takes precedence over the value of CONFIG_IDF_TARGET in sdkconfig.defaults | | -idf.py fails if IDF_TARGET settings don't match in sdkconfig, CMakeCache.txt, and the environment | | +can build with phy_init_data | test_build.py::test_build_skdconfig_phy_init_data | +can build with ethernet component disabled | | moved to test_apps/system/build_test/sdkconfig.ci.ethernet_disabled +Compiler flags on build command line are taken into account | test_build.py::test_build_compiler_flag_in_source_file | +Compiler flags cannot be overwritten | test_build.py::test_build_compiler_flags_no_overwriting | +Can override IDF_TARGET from environment | test_non_default_target.py::test_target_from_environment_cmake | +Can set target using idf.py -D | test_non_default_target.py::test_target_using_D_parameter | +Can set target using -D as subcommand parameter for idf.py | test_non_default_target.py::test_target_using_D_parameter | +Can set target using idf.py set-target | test_non_default_target.py::test_target_using_settarget_parameter | +idf.py understands alternative target names | test_non_default_target.py::test_target_using_settarget_parameter_alternative_name | +Can guess target from sdkconfig, if CMakeCache does not exist | test_non_default_target.py::test_target_using_settarget_parameter | +Can set the default target using sdkconfig.defaults | test_non_default_target.py::test_target_using_sdkconfig | +IDF_TARGET takes precedence over the value of CONFIG_IDF_TARGET in sdkconfig.defaults | test_non_default_target.py::test_target_precedence | +idf.py fails if IDF_TARGET settings don't match in sdkconfig, CMakeCache.txt, and the environment | test_non_default_target.py::test_target_from_environment_idf_py | Setting EXTRA_COMPONENT_DIRS works | | Non-existent paths in EXTRA_COMPONENT_DIRS are not allowed | | Component names may contain spaces | | diff --git a/tools/test_build_system/conftest.py b/tools/test_build_system/conftest.py index 2236d4ca0a..928adb5388 100644 --- a/tools/test_build_system/conftest.py +++ b/tools/test_build_system/conftest.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import datetime import logging @@ -51,7 +51,8 @@ def fixture_session_work_dir(request: FixtureRequest) -> typing.Generator[Path, logging.debug(f'created temporary work directory: {work_dir}') clean_dir = work_dir - yield Path(work_dir) + # resolve allows to use relative paths with --work-dir option + yield Path(work_dir).resolve() if clean_dir: logging.debug(f'cleaning up {clean_dir}') @@ -138,6 +139,6 @@ def fixture_default_idf_env() -> EnvDict: @pytest.fixture def idf_py(default_idf_env: EnvDict) -> IdfPyFunc: - def result(*args: str) -> subprocess.CompletedProcess: - return run_idf_py(*args, env=default_idf_env, workdir=os.getcwd()) # type: ignore + def result(*args: str, check: bool = True) -> subprocess.CompletedProcess: + return run_idf_py(*args, env=default_idf_env, workdir=os.getcwd(), check=check) # type: ignore return result diff --git a/tools/test_build_system/test_build.py b/tools/test_build_system/test_build.py index 48e6fdb1be..72e03ef5ee 100644 --- a/tools/test_build_system/test_build.py +++ b/tools/test_build_system/test_build.py @@ -3,15 +3,15 @@ import logging import os -import re import sys +import textwrap import typing from pathlib import Path -from typing import List, Pattern, Union +from typing import List, Union import pytest -from test_build_system_helpers import (APP_BINS, BOOTLOADER_BINS, PARTITION_BIN, EnvDict, IdfPyFunc, get_idf_build_env, - replace_in_file, run_cmake) +from test_build_system_helpers import (APP_BINS, BOOTLOADER_BINS, PARTITION_BIN, EnvDict, IdfPyFunc, append_to_file, + check_file_contains, get_idf_build_env, replace_in_file, run_cmake) def run_cmake_and_build(*cmake_args: str, env: typing.Optional[EnvDict] = None) -> None: @@ -29,15 +29,6 @@ def assert_built(paths: Union[List[str], List[Path]]) -> None: assert os.path.exists(path) -def check_file_contains(filename: Union[str, Path], what: Union[str, Pattern]) -> None: - with open(filename, 'r', encoding='utf-8') as f: - data = f.read() - if isinstance(what, str): - assert what in data - else: - assert re.search(what, data) is not None - - def test_build_alternative_directories(idf_py: IdfPyFunc, session_work_dir: Path, test_app_copy: Path) -> None: logging.info('Moving BUILD_DIR_BASE out of tree') alt_build_dir = session_work_dir / 'alt_build' @@ -114,3 +105,31 @@ def test_build_with_cmake_and_idf_path_unset(idf_py: IdfPyFunc, test_app_copy: P replace_in_file('CMakeLists.txt', '{IDF_PATH}', '{ci_idf_path}') run_cmake_and_build('-G', 'Ninja', '-D', 'ci_idf_path={}'.format(idf_path), '..', env=env) assert_built(BOOTLOADER_BINS + APP_BINS + PARTITION_BIN) + + +def test_build_skdconfig_phy_init_data(idf_py: IdfPyFunc, test_app_copy: Path) -> None: + logging.info('can build with phy_init_data') + (test_app_copy / 'sdkconfig.defaults').touch() + (test_app_copy / 'sdkconfig.defaults').write_text('CONFIG_ESP32_PHY_INIT_DATA_IN_PARTITION=y') + idf_py('reconfigure') + idf_py('build') + assert_built(BOOTLOADER_BINS + APP_BINS + PARTITION_BIN + ['build/phy_init_data.bin']) + + +def test_build_compiler_flag_in_source_file(idf_py: IdfPyFunc, test_app_copy: Path) -> None: + logging.info('Compiler flags on build command line are taken into account') + compiler_flag = textwrap.dedent(""" + #ifndef USER_FLAG + #error "USER_FLAG is not defined!" + #endif + """) + append_to_file((test_app_copy / 'main' / 'build_test_app.c'), compiler_flag) + idf_py('build', '-DCMAKE_C_FLAGS=-DUSER_FLAG') + + +@pytest.mark.usefixtures('test_app_copy') +def test_build_compiler_flags_no_overwriting(idf_py: IdfPyFunc) -> None: + logging.info('Compiler flags cannot be overwritten') + # If the compiler flags are overriden, the following build command will + # cause issues at link time. + idf_py('build', '-DCMAKE_C_FLAGS=', '-DCMAKE_CXX_FLAGS=') diff --git a/tools/test_build_system/test_build_system_helpers/__init__.py b/tools/test_build_system/test_build_system_helpers/__init__.py index 8cb0da0c18..b5ee5fbe3c 100644 --- a/tools/test_build_system/test_build_system_helpers/__init__.py +++ b/tools/test_build_system/test_build_system_helpers/__init__.py @@ -2,12 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 from .build_constants import ALL_ARTIFACTS, APP_BINS, BOOTLOADER_BINS, JSON_METADATA, PARTITION_BIN from .editing import append_to_file, replace_in_file -from .idf_utils import EXT_IDF_PATH, EnvDict, IdfPyFunc, get_idf_build_env, run_cmake, run_idf_py +from .idf_utils import EXT_IDF_PATH, EnvDict, IdfPyFunc, check_file_contains, get_idf_build_env, run_cmake, run_idf_py from .snapshot import Snapshot, get_snapshot __all__ = [ 'append_to_file', 'replace_in_file', 'get_idf_build_env', 'run_idf_py', 'EXT_IDF_PATH', 'EnvDict', 'IdfPyFunc', 'Snapshot', 'get_snapshot', 'run_cmake', 'APP_BINS', 'BOOTLOADER_BINS', - 'PARTITION_BIN', 'JSON_METADATA', 'ALL_ARTIFACTS' + 'PARTITION_BIN', 'JSON_METADATA', 'ALL_ARTIFACTS', 'check_file_contains' ] diff --git a/tools/test_build_system/test_build_system_helpers/idf_utils.py b/tools/test_build_system/test_build_system_helpers/idf_utils.py index face6d7daf..61d347536a 100644 --- a/tools/test_build_system/test_build_system_helpers/idf_utils.py +++ b/tools/test_build_system/test_build_system_helpers/idf_utils.py @@ -2,11 +2,13 @@ # SPDX-License-Identifier: Apache-2.0 import logging import os +import re import shutil import subprocess import sys import typing from pathlib import Path +from typing import Pattern, Union try: EXT_IDF_PATH = os.environ['IDF_PATH'] # type: str @@ -57,13 +59,15 @@ def get_idf_build_env(idf_path: str) -> EnvDict: def run_idf_py(*args: str, env: typing.Optional[EnvDict] = None, idf_path: typing.Optional[typing.Union[str,Path]] = None, - workdir: typing.Optional[str] = None) -> subprocess.CompletedProcess: + workdir: typing.Optional[str] = None, + check: bool = True) -> subprocess.CompletedProcess: """ Run idf.py command with given arguments, raise an exception on failure :param args: arguments to pass to idf.py :param env: environment variables to run the build with; if not set, the default environment is used :param idf_path: path to the IDF copy to use; if not set, IDF_PATH from the 'env' argument is used :param workdir: directory where to run the build; if not set, the current directory is used + :param check: check process exits with a zero exit code, if false all retvals are accepted without failing the test """ env_dict = dict(**os.environ) if env is not None: @@ -86,7 +90,7 @@ def run_idf_py(*args: str, logging.debug('running {} in {}'.format(' '.join(cmd), workdir)) return subprocess.run( cmd, env=env_dict, cwd=workdir, - check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + check=check, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8', errors='backslashreplace') @@ -107,3 +111,12 @@ def run_cmake(*cmake_args: str, env: typing.Optional[EnvDict] = None) -> None: subprocess.check_call( cmd, env=env, cwd=workdir, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def check_file_contains(filename: Union[str, Path], what: Union[str, Pattern]) -> None: + with open(filename, 'r', encoding='utf-8') as f: + data = f.read() + if isinstance(what, str): + assert what in data + else: + assert re.search(what, data) is not None diff --git a/tools/test_build_system/test_non_default_target.py b/tools/test_build_system/test_non_default_target.py new file mode 100644 index 0000000000..0d0cbe2af7 --- /dev/null +++ b/tools/test_build_system/test_non_default_target.py @@ -0,0 +1,104 @@ +# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import logging +import shutil +from pathlib import Path + +import pytest +from test_build_system_helpers import EnvDict, IdfPyFunc, check_file_contains, run_cmake + +ESP32S2_TARGET = 'esp32s2' +ESP32_TARGET = 'esp32' + + +def clean_app(test_app_copy: Path) -> None: + shutil.rmtree(test_app_copy / 'build') + (test_app_copy / 'sdkconfig').unlink() + + +@pytest.mark.usefixtures('test_app_copy') +def test_target_from_environment_cmake(default_idf_env: EnvDict) -> None: + logging.info('Can override IDF_TARGET from environment') + env = default_idf_env + env.update({'IDF_TARGET': ESP32S2_TARGET}) + run_cmake('-G', 'Ninja', '..', env=env) + check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET="{}"'.format(ESP32S2_TARGET)) + check_file_contains('build/CMakeCache.txt', 'IDF_TARGET:STRING={}'.format(ESP32S2_TARGET)) + + +def test_target_from_environment_idf_py(idf_py: IdfPyFunc, default_idf_env: EnvDict, test_app_copy: Path) -> None: + def reconfigure_and_check_return_values(errmsg: str) -> None: + ret = idf_py('reconfigure', check=False) + assert ret.returncode == 2 + assert errmsg in ret.stderr + + idf_py('set-target', ESP32S2_TARGET) + default_idf_env.update({'IDF_TARGET': ESP32_TARGET}) + + logging.info("idf.py fails if IDF_TARGET settings don't match the environment") + reconfigure_and_check_return_values("Project sdkconfig was generated for target '{}', but environment " + "variable IDF_TARGET is set to '{}'.".format(ESP32S2_TARGET, ESP32_TARGET)) + + logging.info("idf.py fails if IDF_TARGET settings in CMakeCache.txt don't match the environment") + (test_app_copy / 'sdkconfig').write_text('CONFIG_IDF_TARGET="{}"'.format(ESP32_TARGET)) + reconfigure_and_check_return_values("Target settings are not consistent: '{}' in the environment, " + "'{}' in CMakeCache.txt.".format(ESP32_TARGET, ESP32S2_TARGET)) + + logging.info("idf.py fails if IDF_TARGET settings in CMakeCache.txt don't match the sdkconfig") + default_idf_env.pop('IDF_TARGET') + reconfigure_and_check_return_values("Project sdkconfig was generated for target '{}', but CMakeCache.txt " + "contains '{}'.".format(ESP32_TARGET, ESP32S2_TARGET)) + + +def test_target_precedence(idf_py: IdfPyFunc, default_idf_env: EnvDict, test_app_copy: Path) -> None: + logging.info('IDF_TARGET takes precedence over the value of CONFIG_IDF_TARGET in sdkconfig.defaults') + (test_app_copy / 'sdkconfig.defaults').write_text('CONFIG_IDF_TARGET="{}"'.format(ESP32S2_TARGET)) + default_idf_env.update({'IDF_TARGET': ESP32_TARGET}) + idf_py('reconfigure') + check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET="{}"'.format(ESP32_TARGET)) + check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET_{}=y'.format(ESP32_TARGET.upper())) + check_file_contains('build/CMakeCache.txt', 'IDF_TARGET:STRING={}'.format(ESP32_TARGET)) + + +def test_target_using_D_parameter(idf_py: IdfPyFunc, test_app_copy: Path) -> None: + logging.info('Can set target using idf.py -D') + idf_py('-DIDF_TARGET={}'.format(ESP32S2_TARGET), 'reconfigure') + check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET="{}"'.format(ESP32S2_TARGET)) + check_file_contains('build/CMakeCache.txt', 'IDF_TARGET:STRING={}'.format(ESP32S2_TARGET)) + + logging.info('Can set target using -D as subcommand parameter for idf.py') + clean_app(test_app_copy) + idf_py('reconfigure', '-DIDF_TARGET={}'.format(ESP32S2_TARGET)) + check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET="{}"'.format(ESP32S2_TARGET)) + check_file_contains('build/CMakeCache.txt', 'IDF_TARGET:STRING={}'.format(ESP32S2_TARGET)) + + +@pytest.mark.usefixtures('test_app_copy') +def test_target_using_settarget_parameter_alternative_name(idf_py: IdfPyFunc) -> None: + logging.info('idf.py understands alternative target names') + idf_py('set-target', ESP32S2_TARGET.upper()) + check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET="{}"'.format(ESP32S2_TARGET)) + check_file_contains('build/CMakeCache.txt', 'IDF_TARGET:STRING={}'.format(ESP32S2_TARGET)) + + +@pytest.mark.usefixtures('test_app_copy') +def test_target_using_settarget_parameter(idf_py: IdfPyFunc) -> None: + logging.info('Can set target using idf.py set-target') + idf_py('set-target', ESP32S2_TARGET) + check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET="{}"'.format(ESP32S2_TARGET)) + check_file_contains('build/CMakeCache.txt', 'IDF_TARGET:STRING={}'.format(ESP32S2_TARGET)) + + logging.info('Can guess target from sdkconfig, if CMakeCache does not exist') + idf_py('fullclean') + idf_py('reconfigure') + check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET="{}"'.format(ESP32S2_TARGET)) + check_file_contains('build/CMakeCache.txt', 'IDF_TARGET:STRING={}'.format(ESP32S2_TARGET)) + + +def test_target_using_sdkconfig(idf_py: IdfPyFunc, test_app_copy: Path) -> None: + logging.info('Can set the default target using sdkconfig.defaults') + (test_app_copy / 'sdkconfig.defaults').write_text('CONFIG_IDF_TARGET="{}"'.format(ESP32S2_TARGET)) + idf_py('reconfigure') + check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET="{}"'.format(ESP32S2_TARGET)) + check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET_{}=y'.format(ESP32S2_TARGET.upper()))