From fddc73759ebb6f1b00306671474fca7ef90784be Mon Sep 17 00:00:00 2001 From: Roland Dobai Date: Tue, 1 Feb 2022 19:11:46 +0100 Subject: [PATCH] Tools: Make easier the detection of the list of Python features --- .gitlab/CODEOWNERS | 2 - .gitlab/ci/rules.yml | 5 ++ .pre-commit-config.yaml | 8 +++ docs/en/api-guides/tools/idf-tools.rst | 4 +- tools/ci/check_requirement_files.py | 58 +++++++++++++++++++ tools/ci/executable-list.txt | 1 + tools/idf_tools.py | 2 +- tools/requirements.json | 17 ++++++ .../requirements/requirements.core.txt | 0 .../requirements/requirements.gdbgui.txt | 0 tools/requirements_schema.json | 52 +++++++++++++++++ .../test_idf_tools_python_env.py | 4 +- 12 files changed, 146 insertions(+), 7 deletions(-) create mode 100755 tools/ci/check_requirement_files.py create mode 100644 tools/requirements.json rename requirements.core.txt => tools/requirements/requirements.core.txt (100%) rename requirements.gdbgui.txt => tools/requirements/requirements.gdbgui.txt (100%) create mode 100644 tools/requirements_schema.json diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS index 2d264162d6..8986617f85 100644 --- a/.gitlab/CODEOWNERS +++ b/.gitlab/CODEOWNERS @@ -201,8 +201,6 @@ /tools/unit-test-app/ @esp-idf-codeowners/system @esp-idf-codeowners/tools -requirements.*.txt @esp-idf-codeowners/tools - # sort-order-reset /components/**/test_apps/**/*.py @esp-idf-codeowners/ci @esp-idf-codeowners/tools diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 8b1cbc641b..70321d9736 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -47,6 +47,7 @@ - "tools/cmake/**/*" - "tools/kconfig_new/**/*" - "tools/tools.json" + - "tools/requirements.json" - "tools/ci/test_build_system*.sh" .patterns-custom_test: &patterns-custom_test @@ -127,6 +128,10 @@ - "tools/test_idf_tools/**/*" - "tools/install_util.py" + - "tools/requirements/*" + - "tools/requirements.json" + - "tools/requirements_schema.json" + - "tools/mkdfu.py" - "tools/test_mkdfu/**/*" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e401a155ee..6fae306087 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -111,6 +111,14 @@ repos: language: python files: \.(py|c|h|cpp|hpp|ld)$ require_serial: true + - id: check-requirement-files + name: Check requirement files + entry: tools/ci/check_requirement_files.py + additional_dependencies: + - 'jsonschema' + language: python + files: 'tools/requirements.+|tools/requirements/.+' + pass_filenames: false - id: check-tools-files-patterns name: Check tools dir files patterns entry: tools/ci/check_tools_files_patterns.py diff --git a/docs/en/api-guides/tools/idf-tools.rst b/docs/en/api-guides/tools/idf-tools.rst index 64a223780a..a2edac37e5 100644 --- a/docs/en/api-guides/tools/idf-tools.rst +++ b/docs/en/api-guides/tools/idf-tools.rst @@ -104,9 +104,9 @@ Any mirror server can be used provided the URL matches the ``github.com`` downlo * ``check``: For each tool, checks whether the tool is available in the system path and in ``IDF_TOOLS_PATH``. -* ``install-python-env``: Create a Python virtual environment in the ``${IDF_TOOLS_PATH}/python_env`` directory and install there the required Python packages. An optional ``--features`` argument allows one to specify a comma-separated list of features. For each feature a requirements file must exist. For example, feature ``XY`` is a valid feature if ``${IDF_PATH}/requirements.XY.txt`` is an existing file with a list of Python packages to be installed. There is one mandatory ``core`` feature ensuring core functionality of ESP-IDF (build, flash, monitor, debug in console). There can be an arbitrary number of optional features. The selected list of features is stored in ``idf-env.json``. The requirement files contain a list of the desired Python packages to be installed and ``espidf.constraints.*.txt`` downloaded from https://dl.espressif.com and stored in ``${IDF_TOOLS_PATH}`` the package version requirements for a given ESP-IDF version. +* ``install-python-env``: Create a Python virtual environment in the ``${IDF_TOOLS_PATH}/python_env`` directory and install there the required Python packages. An optional ``--features`` argument allows one to specify a comma-separated list of features. For each feature a requirements file must exist. For example, feature ``XY`` is a valid feature if ``${IDF_PATH}/tools/requirements/requirements.XY.txt`` is an existing file with a list of Python packages to be installed. There is one mandatory ``core`` feature ensuring core functionality of ESP-IDF (build, flash, monitor, debug in console). There can be an arbitrary number of optional features. The selected list of features is stored in ``idf-env.json``. The requirement files contain a list of the desired Python packages to be installed and ``espidf.constraints.*.txt`` downloaded from https://dl.espressif.com and stored in ``${IDF_TOOLS_PATH}`` the package version requirements for a given ESP-IDF version. -* ``check-python-dependencies``: Checks if all required Python packages are installed. Packages from ``${IDF_PATH}/requirements.*.txt`` files selected by the feature list of ``idf-env.json`` are checked with the package versions specified in the ``espidf.constraints.*.txt`` file. The constraint file will be downloaded from https://dl.espressif.com if this step hasn't been done already in the last day. +* ``check-python-dependencies``: Checks if all required Python packages are installed. Packages from ``${IDF_PATH}/tools/requirements/requirements.*.txt`` files selected by the feature list of ``idf-env.json`` are checked with the package versions specified in the ``espidf.constraints.*.txt`` file. The constraint file will be downloaded from https://dl.espressif.com if this step hasn't been done already in the last day. .. _idf-tools-install: diff --git a/tools/ci/check_requirement_files.py b/tools/ci/check_requirement_files.py new file mode 100755 index 0000000000..ba3588a089 --- /dev/null +++ b/tools/ci/check_requirement_files.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import json +import os +import re +from typing import Any + +import jsonschema + +IDF_PATH = os.environ['IDF_PATH'] +JSON_PATH = os.path.join(IDF_PATH, 'tools', 'requirements.json') +SCHEMA_PATH = os.path.join(IDF_PATH, 'tools', 'requirements_schema.json') +REQ_DIR = os.path.join(IDF_PATH, 'tools', 'requirements') +RE_FEATURE = re.compile(r'requirements\.(\w+)\.txt') + + +def action_validate(req_obj: Any) -> None: # "Any" because we are checking this in this script + ''' + Check that the parsed JSON object is valid according to the JSON schema provided + ''' + with open(SCHEMA_PATH, 'r') as schema_file: + schema_json = json.load(schema_file) + jsonschema.validate(req_obj, schema_json) + + +def action_check_directory(req_obj: Any) -> None: # "Any" because we are checking this in this script + ''' + Check that all directory items are listed in the JSON file + ''' + features = set(d['name'] for d in req_obj['features']) + features_found = set() + + for file_name in os.listdir(REQ_DIR): + m = re.match(RE_FEATURE, file_name) + if m: + if m.group(1) not in features: + raise RuntimeError(f'Cannot find a feature for {file_name} in {JSON_PATH}') + features_found.add(m.group(1)) + else: + raise RuntimeError(f'{file_name} in {REQ_DIR} doesn\'t match the expected name') + + features_not_found = features - features_found + + if len(features_not_found) > 0: + raise RuntimeError(f'There are no requirements file in {REQ_DIR} for {features_not_found}') + + +def main() -> None: + with open(JSON_PATH, 'r') as f: + req_obj = json.load(f) + action_validate(req_obj) + action_check_directory(req_obj) + + +if __name__ == '__main__': + main() diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 511e85c588..8b90ef1644 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -60,6 +60,7 @@ tools/ci/check_executables.py tools/ci/check_idf_version.sh tools/ci/check_kconfigs.py tools/ci/check_readme_links.py +tools/ci/check_requirement_files.py tools/ci/check_rules_yml.py tools/ci/check_soc_struct_headers.py tools/ci/check_tools_files_patterns.py diff --git a/tools/idf_tools.py b/tools/idf_tools.py index 1670ade1b0..f115a15536 100755 --- a/tools/idf_tools.py +++ b/tools/idf_tools.py @@ -1091,7 +1091,7 @@ def add_and_save_targets(targets_str): # type: (str) -> list[str] def feature_to_requirements_path(feature): # type: (str) -> str - return os.path.join(global_idf_path or '', 'requirements.{}.txt'.format(feature)) + return os.path.join(global_idf_path or '', 'tools', 'requirements', 'requirements.{}.txt'.format(feature)) def add_and_save_features(features_str): # type: (str) -> list[str] diff --git a/tools/requirements.json b/tools/requirements.json new file mode 100644 index 0000000000..8136e39488 --- /dev/null +++ b/tools/requirements.json @@ -0,0 +1,17 @@ +{ + "version": 1, + "features": [ + { + "name": "core", + "description": "Core packages necessary for ESP-IDF", + "optional": false, + "requirement_path": "tools/requirements/requirements.core.txt" + }, + { + "name": "gdbgui", + "description": "Packages for supporting debugging from web browser", + "optional": true, + "requirement_path": "tools/requirements/requirements.gdbgui.txt" + } + ] +} diff --git a/requirements.core.txt b/tools/requirements/requirements.core.txt similarity index 100% rename from requirements.core.txt rename to tools/requirements/requirements.core.txt diff --git a/requirements.gdbgui.txt b/tools/requirements/requirements.gdbgui.txt similarity index 100% rename from requirements.gdbgui.txt rename to tools/requirements/requirements.gdbgui.txt diff --git a/tools/requirements_schema.json b/tools/requirements_schema.json new file mode 100644 index 0000000000..7d8ee01b03 --- /dev/null +++ b/tools/requirements_schema.json @@ -0,0 +1,52 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/espressif/esp-idf/blob/master/tools/requirements_schema.json", + "type": "object", + "properties": { + "version": { + "type": "integer", + "description": "Metadata file version" + }, + "features": { + "type": "array", + "description": "List of features", + "items": { + "$ref": "#/definitions/featInfo" + } + } + }, + "required": [ + "version", + "features" + ], + "definitions": { + "featInfo": { + "type": "object", + "description": "Information about one feature", + "properties": { + "name" : { + "description": "Feature name", + "type": "string" + }, + "description" : { + "description": "A short description of the feature", + "type": "string" + }, + "optional": { + "description": "The feature is optional if the user can choose to not install it", + "type": "boolean" + }, + "requirement_path": { + "description": "Path to the requirements file with Python packages", + "type": "string", + "pattern": "^tools/requirements/requirements\\..+\\.txt$" + } + }, + "required": [ + "name", + "optional", + "requirement_path" + ] + } + } +} diff --git a/tools/test_idf_tools/test_idf_tools_python_env.py b/tools/test_idf_tools/test_idf_tools_python_env.py index 9f602fc92e..7ff528db0c 100644 --- a/tools/test_idf_tools/test_idf_tools_python_env.py +++ b/tools/test_idf_tools/test_idf_tools_python_env.py @@ -18,8 +18,8 @@ IDF_PATH = os.environ.get('IDF_PATH', '../..') TOOLS_DIR = os.environ.get('IDF_TOOLS_PATH') or os.path.expanduser(idf_tools.IDF_TOOLS_PATH_DEFAULT) PYTHON_DIR = os.path.join(TOOLS_DIR, 'python_env') REQ_SATISFIED = 'Python requirements are satisfied' -REQ_CORE = '- {}/requirements.core.txt'.format(IDF_PATH) -REQ_GDBGUI = '- {}/requirements.gdbgui.txt'.format(IDF_PATH) +REQ_CORE = '- {}'.format(os.path.join(IDF_PATH, 'tools', 'requirements', 'requirements.core.txt')) +REQ_GDBGUI = '- {}'.format(os.path.join(IDF_PATH, 'tools', 'requirements', 'requirements.gdbgui.txt')) CONSTR = 'Constraint file: {}/espidf.constraints'.format(TOOLS_DIR)