ci: reimplement build_examples_cmake using {find,build}_apps.py

tools/ci/build_examples_cmake.sh

@@ -1,24 +1,10 @@
 #!/bin/bash
 #
-# Build all examples from the examples directory, out of tree to
+# Build all examples from the examples directory, in BUILD_PATH to
 # ensure they can run when copied to a new directory.
 #
 # Runs as part of CI process.
 #
-# Assumes PWD is an out-of-tree build directory, and will copy examples
-# to individual subdirectories, one by one.
-#
-#
-# Without arguments it just builds all examples
-#
-# With one argument <JOB_NAME> it builds part of the examples. This is a useful for
-#   parallel execution in CI.
-#   <JOB_NAME> must look like this:
-#               <some_text_label>_<num>
-#   It scans .gitlab-ci.yaml to count number of jobs which have name "<some_text_label>_<num>"
-#   It scans the filesystem to count all examples
-#   Based on this, it decides to run qa set of examples.
-#
 
 # -----------------------------------------------------------------------------
 # Safety settings (see https://gist.github.com/ilg-ul/383869cbb01f61a51c4d).
@@ -31,9 +17,6 @@ fi
 set -o errexit # Exit if command failed.
 set -o pipefail # Exit if pipe failed.
 
-# Remove the initial space and instead use '\n'.
-IFS=$'\n\t'
-
 export PATH="$IDF_PATH/tools/ci:$IDF_PATH/tools:$PATH"
 
 # -----------------------------------------------------------------------------
@@ -45,146 +28,76 @@ die() {
 
 [ -z ${IDF_PATH} ] && die "IDF_PATH is not set"
 [ -z ${LOG_PATH} ] && die "LOG_PATH is not set"
+[ -z ${BUILD_PATH} ] && die "BUILD_PATH is not set"
+[ -z ${IDF_TARGET} ] && die "IDF_TARGET is not set"
 [ -d ${LOG_PATH} ] || mkdir -p ${LOG_PATH}
+[ -d ${BUILD_PATH} ] || mkdir -p ${BUILD_PATH}
+
+if [ -z ${CI_NODE_TOTAL} ]; then
+    CI_NODE_TOTAL=1
+    echo "Assuming CI_NODE_TOTAL=${CI_NODE_TOTAL}"
+fi
+if [ -z ${CI_NODE_INDEX} ]; then
+    # Gitlab uses a 1-based index
+    CI_NODE_INDEX=1
+    echo "Assuming CI_NODE_INDEX=${CI_NODE_INDEX}"
+fi
 
 set -o nounset # Exit if variable not set.
 
-echo "build_examples running in ${PWD} for target $IDF_TARGET"
+export REALPATH=realpath
-
+if [ "$(uname -s)" = "Darwin" ]; then
-# only 0 or 1 arguments
+    export REALPATH=grealpath
-[ $# -le 1 ] || die "Have to run as $(basename $0) [<JOB_NAME>]"
-
-export BATCH_BUILD=1
-export V=0 # only build verbose if there's an error
-
-shopt -s lastpipe # Workaround for Bash to use variables in loops (http://mywiki.wooledge.org/BashFAQ/024)
-
-RESULT=0
-FAILED_EXAMPLES=""
-RESULT_ISSUES=22  # magic number result code for issues found
-LOG_SUSPECTED=${LOG_PATH}/common_log.txt
-touch ${LOG_SUSPECTED}
-SDKCONFIG_DEFAULTS_CI=sdkconfig.ci
-
-EXAMPLE_PATHS=$( get_supported_examples.sh $IDF_TARGET | sed "s#^#${IDF_PATH}\/examples\/#g" | awk '{print $0"/CmakeLists.txt"}' )
-echo "All examples found for target $IDF_TARGET:"
-echo $EXAMPLE_PATHS
-
-if [ -z {CI_NODE_TOTAL} ]
-then
-    START_NUM=0
-    END_NUM=999
-else
-    JOB_NUM=${CI_NODE_INDEX}
-    # count number of the jobs
-    NUM_OF_JOBS=${CI_NODE_TOTAL}
-
-    # count number of examples
-    NUM_OF_EXAMPLES=$( echo "${EXAMPLE_PATHS}" | wc -l )
-    [ ${NUM_OF_EXAMPLES} -lt 100 ] && die "NUM_OF_EXAMPLES is bad"
-
-    # separate intervals
-    #57 / 5 == 12
-    NUM_OF_EX_PER_JOB=$(( (${NUM_OF_EXAMPLES} + ${NUM_OF_JOBS} - 1) / ${NUM_OF_JOBS} ))
-    [ -z ${NUM_OF_EX_PER_JOB} ] && die "NUM_OF_EX_PER_JOB is bad"
-
-    # ex.: [0; 12); [12; 24); [24; 36); [36; 48); [48; 60)
-    START_NUM=$(( (${JOB_NUM} - 1) * ${NUM_OF_EX_PER_JOB} ))
-    [ -z ${START_NUM} ] && die "START_NUM is bad"
-
-    END_NUM=$(( ${JOB_NUM} * ${NUM_OF_EX_PER_JOB} ))
-    [ -z ${END_NUM} ] && die "END_NUM is bad"
 fi
 
-build_example () {
+# Convert LOG_PATH and BUILD_PATH to relative, to make the json file less verbose.
-    local ID=$1
+LOG_PATH=$(${REALPATH} --relative-to ${IDF_PATH} ${LOG_PATH})
-    shift
+BUILD_PATH=$(${REALPATH} --relative-to ${IDF_PATH} ${BUILD_PATH})
-    local CMAKELISTS=$1
-    shift
 
-    local EXAMPLE_DIR=$(dirname "${CMAKELISTS}")
+ALL_BUILD_LIST_JSON="${BUILD_PATH}/list.json"
-    local EXAMPLE_NAME=$(basename "${EXAMPLE_DIR}")
+JOB_BUILD_LIST_JSON="${BUILD_PATH}/list_job_${CI_NODE_INDEX}.json"
+mkdir -p "${BUILD_PATH}/example_builds"
 
-    echo "Building ${EXAMPLE_NAME} for ${IDF_TARGET} as ${ID}..."
+echo "build_examples running for target $IDF_TARGET"
-    mkdir -p "example_builds/${IDF_TARGET}/${ID}"
-    cp -r "${EXAMPLE_DIR}" "example_builds/${IDF_TARGET}/${ID}"
-    pushd "example_builds/${IDF_TARGET}/${ID}/${EXAMPLE_NAME}"
-        # be stricter in the CI build than the default IDF settings
-        export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
-        export EXTRA_CXXFLAGS=${EXTRA_CFLAGS}
 
-        # sdkconfig files are normally not checked into git, but may be present when
+cd ${IDF_PATH}
-        # a developer runs this script locally
-        rm -f sdkconfig
 
-        # If sdkconfig.ci file is present, append it to sdkconfig.defaults,
+# This part of the script produces the same result for all the example build jobs. It may be moved to a separate stage
-        # replacing environment variables
+# (pre-build) later, then the build jobs will receive ${BUILD_LIST_JSON} file as an artifact.
-        if [[ -f "$SDKCONFIG_DEFAULTS_CI" ]]; then
-            cat $SDKCONFIG_DEFAULTS_CI | $IDF_PATH/tools/ci/envsubst.py >> sdkconfig.defaults
-        fi
 
-        # build non-verbose first
+# If changing the work-dir or build-dir format, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py.
-        local BUILDLOG=${LOG_PATH}/ex_${ID}_log.txt
-        touch ${BUILDLOG}
 
-        if [ "$EXAMPLE_NAME" != "idf_as_lib" ]; then
+${IDF_PATH}/tools/find_apps.py examples \
-            idf.py fullclean >>${BUILDLOG} 2>&1 &&
+    -vv \
-            idf.py build >>${BUILDLOG} 2>&1
+    --format json \
-        else
+    --build-system cmake \
-            rm -rf build &&
+    --target ${IDF_TARGET} \
-            ./build-esp32.sh >>${BUILDLOG} 2>&1
+    --recursive \
-        fi ||
+    --exclude examples/build_system/idf_as_lib \
-        {
+    --work-dir "${BUILD_PATH}/@f/@w/@t" \
-            RESULT=$?; FAILED_EXAMPLES+=" ${EXAMPLE_NAME}" ;
+    --build-dir build \
-        }
+    --build-log "${LOG_PATH}/@f.txt" \
+    --output ${ALL_BUILD_LIST_JSON} \
+    --config 'sdkconfig.ci=default' \
+    --config 'sdkconfig.ci.*=' \
+    --config '=default' \
 
-        cat ${BUILDLOG}
+# --config rules above explained:
-    popd
+# 1. If sdkconfig.ci exists, use it build the example with configuration name "default"
+# 2. If sdkconfig.ci.* exists, use it to build the "*" configuration
+# 3. If none of the above exist, build the default configuration under the name "default"
 
-    grep -i "error\|warning" "${BUILDLOG}" 2>&1 | grep -v "error.c.obj" >> "${LOG_SUSPECTED}" || :
+# The part below is where the actual builds happen
-}
 
-EXAMPLE_NUM=0
+${IDF_PATH}/tools/build_apps.py \
+    -vv \
+    --format json \
+    --keep-going \
+    --parallel-count ${CI_NODE_TOTAL} \
+    --parallel-index ${CI_NODE_INDEX} \
+    --output-build-list ${JOB_BUILD_LIST_JSON} \
+    ${ALL_BUILD_LIST_JSON}\
 
-echo "Current job will build example ${START_NUM} - ${END_NUM}"
 
-for EXAMPLE_PATH in ${EXAMPLE_PATHS}
+# Check for build warnings
-do
+${IDF_PATH}/tools/ci/check_build_warnings.py -vv ${JOB_BUILD_LIST_JSON}
-    if [[ $EXAMPLE_NUM -lt $START_NUM || $EXAMPLE_NUM -ge $END_NUM ]]
-    then
-        EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 ))
-        continue
-    fi
-    echo ">>> example [ ${EXAMPLE_NUM} ] - $EXAMPLE_PATH"
-
-    build_example "${EXAMPLE_NUM}" "${EXAMPLE_PATH}"
-
-    EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 ))
-done
-
-# show warnings
-echo -e "\nFound issues:"
-
-#       Ignore the next messages:
-# "error.o" or "-Werror" in compiler's command line
-# "reassigning to symbol" or "changes choice state" in sdkconfig
-# 'Compiler and toochain versions is not supported' from crosstool_version_check.cmake
-IGNORE_WARNS="\
-library/error\.o\
-\|\ -Werror\
-\|error\.d\
-\|reassigning to symbol\
-\|changes choice state\
-\|crosstool_version_check\.cmake\
-"
-
-sort -u "${LOG_SUSPECTED}" | grep -v "${IGNORE_WARNS}" \
-    && RESULT=$RESULT_ISSUES \
-    || echo -e "\tNone"
-
-[ -z ${FAILED_EXAMPLES} ] || echo -e "\nThere are errors in the next examples: $FAILED_EXAMPLES"
-[ $RESULT -eq 0 ] || echo -e "\nFix all warnings and errors above to pass the test!"
-
-echo -e "\nReturn code = $RESULT"
-
-exit $RESULT

tools/ci/config/build.yml

@@ -159,16 +159,20 @@ build_examples_make:
   artifacts:
     when: always
     paths:
-      - build_examples_cmake/*/*/*/*/build/*.bin
+      - build_examples/list.json
-      - build_examples_cmake/*/*/*/*/sdkconfig
+      - build_examples/list_job_*.json
-      - build_examples_cmake/*/*/*/*/build/*.elf
+      - build_examples/*/*/*/sdkconfig
-      - build_examples_cmake/*/*/*/*/build/*.map
+      - build_examples/*/*/*/build/*.bin
-      - build_examples_cmake/*/*/*/*/build/flasher_args.json
+      - build_examples/*/*/*/build/*.elf
-      - build_examples_cmake/*/*/*/*/build/bootloader/*.bin
+      - build_examples/*/*/*/build/*.map
+      - build_examples/*/*/*/build/flasher_args.json
+      - build_examples/*/*/*/build/bootloader/*.bin
+      - build_examples/*/*/*/build/partition_table/*.bin
       - $LOG_PATH
     expire_in: 3 days
   variables:
-    LOG_PATH: "$CI_PROJECT_DIR/log_examples_cmake"
+    LOG_PATH: "$CI_PROJECT_DIR/log_examples"
+    BUILD_PATH: "$CI_PROJECT_DIR/build_examples"
   only:
     variables:
       - $BOT_TRIGGER_WITH_LABEL == null
@@ -179,10 +183,7 @@ build_examples_make:
   script:
     # it's not possible to build 100% out-of-tree and have the "artifacts"
     # mechanism work, but this is the next best thing
-    - rm -rf build_examples_cmake
+    - mkdir -p ${BUILD_PATH}
-    - mkdir build_examples_cmake
-    - cd build_examples_cmake
-    # build some of examples
     - mkdir -p ${LOG_PATH}
     - ${IDF_PATH}/tools/ci/build_examples_cmake.sh