diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index ee1ed88944..4dfca837f1 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -213,6 +213,14 @@ test_mkuf2: - cd ${IDF_PATH}/tools/test_mkuf2 - ./test_mkuf2.py +test_sbom: + extends: + - .host_test_template + - .rules:patterns:sbom + script: + - cd ${IDF_PATH}/tools/test_sbom + - pytest + test_autocomplete: extends: - .host_test_template diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 31725449d8..436aada638 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -54,6 +54,9 @@ - "tools/ci/ci_build_apps.py" - "tools/test_build_system/**/*" +.patterns-sbom: &patterns-sbom + - "tools/test_sbom/*" + .patterns-custom_test: &patterns-custom_test - "tools/ci/python_packages/gitlab_api.py" - "tools/ci/python_packages/tiny_test_fw/**/*" @@ -445,6 +448,14 @@ - <<: *if-dev-push changes: *patterns-sonarqube-files +.rules:patterns:sbom: + rules: + - <<: *if-protected + - <<: *if-dev-push + changes: *patterns-sbom + - <<: *if-dev-push + changes: *patterns-submodule + # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # DO NOT place comments or maintain any code from this line # diff --git a/.gitmodules b/.gitmodules index b3b9edd1be..0b4bc720b9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,6 +2,26 @@ # All the relative URL paths are intended to be GitHub ones # For Espressif's public projects please use '../../espressif/proj', not a '../proj' # +# Submodules SBOM information +# --------------------------- +# Submodules, which are used directly and not forked into espressif namespace should +# contain SBOM information here. Other submodules should have the SBOM manifest file +# included in the root of their project's repository. +# +# The sbom-hash entry records the submodule's checkout SHA as presented in git-tree +# commit object. For example spiffs submodule +# +# $ git ls-tree HEAD components/spiffs/spiffs +# 160000 commit 0dbb3f71c5f6fae3747a9d935372773762baf852 components/spiffs/spiffs +# +# The hash can be also obtained with git submodule command +# +# $ git submodule status components/spiffs/spiffs +# 0dbb3f71c5f6fae3747a9d935372773762baf852 components/spiffs/spiffs (0.2-255-g0dbb3f71c5f6) +# +# The submodule SHA recorded here has to match with SHA, which is presented in git-tree. +# This is checked by CI. Also please don't forget to update the submodule version +# if you are changing the sbom-hash. This is important for SBOM generation. [submodule "components/bt/controller/lib_esp32"] path = components/bt/controller/lib_esp32 @@ -10,14 +30,31 @@ [submodule "components/bootloader/subproject/components/micro-ecc/micro-ecc"] path = components/bootloader/subproject/components/micro-ecc/micro-ecc url = ../../kmackay/micro-ecc.git + sbom-version = 1.0 + sbom-cpe = cpe:2.3:a:micro-ecc_project:micro-ecc:{}:*:*:*:*:*:*:* + sbom-supplier = Person: Ken MacKay + sbom-url = https://github.com/kmackay/micro-ecc + sbom-description = A small and fast ECDH and ECDSA implementation for 8-bit, 32-bit, and 64-bit processors + sbom-hash = d037ec89546fad14b5c4d5456c2e23a71e554966 [submodule "components/spiffs/spiffs"] path = components/spiffs/spiffs url = ../../pellepl/spiffs.git + sbom-version = 0.2-255-g0dbb3f71c5f6 + sbom-supplier = Person: Peter Andersson + sbom-url = https://github.com/pellepl/spiffs + sbom-description = Wear-leveled SPI flash file system for embedded devices + sbom-hash = 0dbb3f71c5f6fae3747a9d935372773762baf852 [submodule "components/json/cJSON"] path = components/json/cJSON url = ../../DaveGamble/cJSON.git + sbom-version = 1.7.15 + sbom-cpe = cpe:2.3:a:cjson_project:cjson:{}:*:*:*:*:*:*:* + sbom-supplier = Person: Dave Gamble + sbom-url = https://github.com/DaveGamble/cJSON + sbom-description = Ultralightweight JSON parser in ANSI C + sbom-hash = d348621ca93571343a56862df7de4ff3bc9b5667 [submodule "components/mbedtls/mbedtls"] path = components/mbedtls/mbedtls @@ -34,10 +71,21 @@ [submodule "components/protobuf-c/protobuf-c"] path = components/protobuf-c/protobuf-c url = ../../protobuf-c/protobuf-c.git + sbom-version = 1.4.1 + sbom-cpe = cpe:2.3:a:protobuf-c_project:protobuf-c:{}:*:*:*:*:*:*:* + sbom-supplier = Organization: protobuf-c community + sbom-url = https://github.com/protobuf-c/protobuf-c + sbom-description = Protocol Buffers implementation in C + sbom-hash = abc67a11c6db271bedbb9f58be85d6f4e2ea8389 [submodule "components/unity/unity"] path = components/unity/unity url = ../../ThrowTheSwitch/Unity.git + sbom-version = v2.4.3-51-g7d2bf62b7e6a + sbom-supplier = Organization: ThrowTheSwitch community + sbom-url = https://github.com/ThrowTheSwitch/Unity + sbom-description = Simple Unit Testing for C + sbom-hash = 7d2bf62b7e6afaf38153041a9d53c21aeeca9a25 [submodule "components/bt/host/nimble/nimble"] path = components/bt/host/nimble/nimble @@ -50,6 +98,11 @@ [submodule "components/cmock/CMock"] path = components/cmock/CMock url = ../../ThrowTheSwitch/CMock.git + sbom-version = v2.5.2-2-geeecc49ce8af + sbom-supplier = Organization: ThrowTheSwitch community + sbom-url = https://github.com/ThrowTheSwitch/CMock + sbom-description = CMock - Mock/stub generator for C + sbom-hash = eeecc49ce8af123cf8ad40efdb9673e37b56230f [submodule "components/openthread/openthread"] path = components/openthread/openthread diff --git a/tools/test_sbom/pytest.ini b/tools/test_sbom/pytest.ini new file mode 100644 index 0000000000..d95e773e5c --- /dev/null +++ b/tools/test_sbom/pytest.ini @@ -0,0 +1,12 @@ +[pytest] +addopts = -s -p no:pytest_embedded + +# log related +log_cli = True +log_cli_level = INFO +log_cli_format = %(asctime)s %(levelname)s %(message)s +log_cli_date_format = %Y-%m-%d %H:%M:%S + +## log all to `system-out` when case fail +junit_logging = stdout +junit_log_passing_tests = False diff --git a/tools/test_sbom/test_submodules.py b/tools/test_sbom/test_submodules.py new file mode 100644 index 0000000000..0bc161bcfb --- /dev/null +++ b/tools/test_sbom/test_submodules.py @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import os +from subprocess import run +from typing import Dict, List + + +def run_cmd(cmd: List[str]) -> str: + """Simple helper to run command and return it's stdout.""" + proc = run(cmd, capture_output=True, check=True, text=True) + return proc.stdout.strip() + + +def get_gitwdir() -> str: + """Return absolute path to the current git working tree.""" + return run_cmd(['git', 'rev-parse', '--show-toplevel']) + + +def get_submodules_info() -> List[Dict[str,str]]: + """Return list of submodules, where each submodule is represented + as dictionary with name, path and hash keys.""" + cmd = ['git', 'submodule', '--quiet', 'foreach','echo "$name,$sm_path,$sha1"'] + out = run_cmd(cmd) + submodules = [] + for line in out.splitlines(): + name, sm_path, sha1 = line.split(',') + submodules += [{'name': name, 'path': sm_path, 'hash': sha1}] + + return submodules + + +def get_submodules_config() -> Dict[str,str]: + """Return dictionary, where key is variable name and value + is variable value in git's --list(dot) format. Only variables + starting with "submodule." are returned and this prefix is removed + to make it simple to match against the submodule info dictionary.""" + gitmodules_fn = os.path.join(get_gitwdir(), '.gitmodules') + gitmodules_data = run_cmd(['git', 'config', '--list', '--file', gitmodules_fn]) + prefix = 'submodule.' + config = {} + for line in gitmodules_data.splitlines(): + var, val = line.split('=', maxsplit=1) + if not var.startswith(prefix): + continue + # remove "submodule." prefix + var = var[len(prefix):] + config[var] = val + + return config + + +def test_sha() -> None: + """ Check that submodule SHA1 in git-tree and .gitmodules match + if sbom-hash variable is available in the .gitmodules file. + """ + submodules = get_submodules_info() + config = get_submodules_config() + + for submodule in submodules: + sbom_hash = config.get(submodule['name'] + '.sbom-hash') + if not sbom_hash: + continue + msg = (f'Submodule \"{submodule["name"]}\" SHA \"{submodule["hash"]}\" in git ' + f'tree does not match SHA \"{sbom_hash}\" recorded in .gitmodules. ' + f'Please update \"sbom-hash\" in .gitmodules for \"{submodule["name"]}\" ' + f'and also please do not forget to update version and other submodule ' + f'information if necessary. It is important to keep this information ' + f'up-to-date for SBOM generation.') + assert submodule['hash'] == sbom_hash, msg