From 5df4decf07628ad34c78a47c11f98d3e4301ab25 Mon Sep 17 00:00:00 2001 From: Frantisek Hrbata Date: Tue, 5 Dec 2023 10:06:05 +0100 Subject: [PATCH] feat: use esp-idf-sbom pre-commit plugin Currently sbom manifest is checked only in .gitmodules and this check is done in pre-commit and also in CI. Meaning it's running three times(pre-commit before push if user has it enabled, in CI as there is the pre-commit run again and again with test in CI). Since esp-idf-sbom contains a full manifest validation support and pre-commit plugin for it, let's use it. This removes all the current sbom testing and replaces it with a signle pre-commit plugin which validates all manifests files(sbom.yml, idf_component.yml, .gitmodules and also referenced manifests) in repository. Note that this checks all manifests, not only ones which were modified. The check is reasonably fast though, so it should not cause any problem. The reason for validating all manifest files is that we want to make sure that the sbom information in .gitmodules is updated too and that the hash recorded in .gitmodules is up-to-date. Meaning submodule update would not trigger this plugin, because no manifest was changed. Signed-off-by: Frantisek Hrbata --- .gitlab/ci/host-test.yml | 2 - .gitlab/ci/rules.yml | 2 - .pre-commit-config.yaml | 10 ++-- tools/test_sbom/pytest.ini | 12 ----- tools/test_sbom/test_submodules.py | 74 ------------------------------ 5 files changed, 4 insertions(+), 96 deletions(-) delete mode 100644 tools/test_sbom/pytest.ini delete mode 100644 tools/test_sbom/test_submodules.py diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index ec6988e314..b43855f70c 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -214,8 +214,6 @@ test_tools: - pytest --noconftest test_idf_qemu.py --junitxml=${IDF_PATH}/XUNIT_IDF_PY_QEMU.xml || stat=1 - cd ${IDF_PATH}/tools/test_mkdfu - pytest --noconftest test_mkdfu.py --junitxml=${IDF_PATH}/XUNIT_MKDFU.xml || stat=1 - - cd ${IDF_PATH}/tools/test_sbom - - pytest --junitxml=${IDF_PATH}/XUNIT_SBOM.xml || stat=1 - cd ${IDF_PATH} - shellcheck -s sh tools/detect_python.sh || stat=1 - shellcheck -s bash tools/detect_python.sh || stat=1 diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 76f471adad..419faeb38d 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -147,8 +147,6 @@ - "tools/test_idf_tools/**/*" - "tools/install_util.py" - - "tools/test_sbom/*" - - "tools/requirements/*" - "tools/requirements.json" - "tools/requirements_schema.json" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9b522cc4d9..92d7d9d5d3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -177,12 +177,6 @@ repos: always_run: true pass_filenames: false require_serial: true - - id: submodule-sbom-hash-check - name: Check if sbom-hash values for submodules in .gitmodules match submodules checkout hash in git tree - entry: python tools/test_sbom/test_submodules.py - language: python - always_run: true - pass_filenames: false - id: cleanup-ignore-lists name: Remove non-existing patterns from ignore lists entry: tools/ci/cleanup_ignore_lists.py @@ -221,3 +215,7 @@ repos: name: shellcheck dash (export.sh) args: ['--shell', 'dash', '-x'] files: 'export.sh' + - repo: https://github.com/espressif/esp-idf-sbom.git + rev: v0.11.0 + hooks: + - id: validate-sbom-manifest diff --git a/tools/test_sbom/pytest.ini b/tools/test_sbom/pytest.ini deleted file mode 100644 index d95e773e5c..0000000000 --- a/tools/test_sbom/pytest.ini +++ /dev/null @@ -1,12 +0,0 @@ -[pytest] -addopts = -s -p no:pytest_embedded - -# log related -log_cli = True -log_cli_level = INFO -log_cli_format = %(asctime)s %(levelname)s %(message)s -log_cli_date_format = %Y-%m-%d %H:%M:%S - -## log all to `system-out` when case fail -junit_logging = stdout -junit_log_passing_tests = False diff --git a/tools/test_sbom/test_submodules.py b/tools/test_sbom/test_submodules.py deleted file mode 100644 index 85f76ab850..0000000000 --- a/tools/test_sbom/test_submodules.py +++ /dev/null @@ -1,74 +0,0 @@ -# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import os -from subprocess import run -from typing import Dict, List - - -def run_cmd(cmd: List[str]) -> str: - """Simple helper to run command and return it's stdout.""" - proc = run(cmd, capture_output=True, check=True, text=True) - return proc.stdout.strip() - - -def get_gitwdir() -> str: - """Return absolute path to the current git working tree.""" - return run_cmd(['git', 'rev-parse', '--show-toplevel']) - - -def get_submodules_config() -> Dict[str,Dict[str,str]]: - """Return dictionary, where key is submodule name and value - is a dictionary with variable:value pairs.""" - gitmodules_fn = os.path.join(get_gitwdir(), '.gitmodules') - gitmodules_data = run_cmd(['git', 'config', '--list', '--file', gitmodules_fn]) - prefix = 'submodule.' - config: Dict[str, Dict[str,str]] = {} - for line in gitmodules_data.splitlines(): - if not line.startswith(prefix): - continue - splitted = line.split('=', maxsplit=1) - if len(splitted) != 2: - continue - section, val = splitted - # remove "submodule." prefix - section = section[len(prefix):] - # split section into module name and variable - splitted = section.rsplit('.', maxsplit=1) - if len(splitted) != 2: - continue - module_name, var = splitted - if module_name not in config: - config[module_name] = {} - config[module_name][var] = val - - return config - - -def test_sha() -> None: - """ Check that submodule SHA in git-tree and .gitmodules match - if sbom-hash variable is available in the .gitmodules file. - """ - submodules = get_submodules_config() - - for name, variables in submodules.items(): - sbom_hash = variables.get('sbom-hash') - if not sbom_hash: - continue - module_path = variables.get('path') - if not module_path: - continue - output = run_cmd(['git', 'ls-tree', 'HEAD', module_path]) - if not output: - continue - module_hash = output.split()[2] - msg = (f'Submodule \"{name}\" SHA \"{module_hash}\" in git ' - f'tree does not match SHA \"{sbom_hash}\" recorded in .gitmodules. ' - f'Please update \"sbom-hash\" in .gitmodules for \"{name}\" ' - f'and also please do not forget to update version and other submodule ' - f'information if necessary. It is important to keep this information ' - f'up-to-date for SBOM generation.') - assert module_hash == sbom_hash, msg - - -if __name__ == '__main__': - test_sha()