Merge branch 'ci/use_cache_with_reqirements_txts' into 'master'

ci: new pip cache policy and pytest requirements

Closes IDF-4604

See merge request espressif/esp-idf!18377

.gitlab-ci.yml

@@ -1,4 +1,5 @@
 stages:
+  - upload_cache
   - pre_check
   - build
   - assign_test
@@ -74,7 +75,6 @@ variables:
   TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git"
   CI_AUTO_TEST_SCRIPT_REPO_URL: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/auto_test_script.git"
   CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v4.1"
-  PYTEST_EMBEDDED_VERSION: "0.7.1"
 
   # cache python dependencies
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
@@ -101,7 +101,7 @@ cache:
   paths:
     - .cache/pip
   # pull only for most of the use cases since it's cache dir.
-  # Only set "pull-push" policy for "scan_test" since it would **possibly** install all pypi packages
+  # Only set "push" policy for "upload_cache" stage jobs since it would install all pypi packages
   policy: pull
 
 .setup_tools_unless_target_test: &setup_tools_unless_target_test |
@@ -209,20 +209,12 @@ before_script:
     - *setup_tools_unless_target_test
     - fetch_submodules
     - *download_test_python_contraint_file
-    - $IDF_PATH/tools/idf_tools.py install-python-env
+    - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest
     # TODO: remove this, IDFCI-1207
     - pip install esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
-    - pip install
-      "pytest-embedded-serial-esp~=$PYTEST_EMBEDDED_VERSION"
-      "pytest-embedded-idf~=$PYTEST_EMBEDDED_VERSION"
-      "pytest-embedded-qemu~=$PYTEST_EMBEDDED_VERSION"
-      pytest-rerunfailures
-      scapy
-      websocket-client
-      netifaces
-      -r tools/esp_prov/requirements.txt
     - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
     - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
+    - eval "$($IDF_PATH/tools/idf_tools.py export)"  # use idf venv instead
 
 default:
   retry:
@@ -235,6 +227,7 @@ default:
 
 include:
   - '.gitlab/ci/rules.yml'
+  - '.gitlab/ci/upload_cache.yml'
   - '.gitlab/ci/docs.yml'
   - '.gitlab/ci/static-code-analysis.yml'
   - '.gitlab/ci/pre_check.yml'

.gitlab/ci/host-test.yml

@@ -7,7 +7,11 @@
   tags:
     - host_test
   dependencies: []
-  needs: []  # run host_test jobs immediately
+  needs:
+    - job: upload-pip-cache-shiny
+      optional: true  # run host_test jobs immediately, only after upload cache
+    - job: upload-pip-cache-brew
+      optional: true  # run host_test jobs immediately, only after upload cache
 
 test_nvs_on_host:
   extends: .host_test_template

.gitlab/ci/pre_check.yml

@@ -160,13 +160,6 @@ scan_tests:
       - $TEST_APPS_OUTPUT_DIR
       - $COMPONENT_UT_OUTPUT_DIR
     expire_in: 1 week
-  cache:
-    key: pip-cache
-    paths:
-      - .cache/pip
-    # pull only for most of the use cases since it's cache dir.
-    # Only set "pull-push" policy for "scan_test" since it would **possibly** install all pypi packages
-    policy: pull-push
   variables:
     EXAMPLE_TEST_DIR: ${CI_PROJECT_DIR}/examples
     EXAMPLE_TEST_OUTPUT_DIR: ${CI_PROJECT_DIR}/examples/test_configs

.gitlab/ci/rules.yml

@@ -12,6 +12,10 @@
   - "components/**/Kconfig"
   - "components/**/CMakeList.txt"
 
+.patterns-python-cache: &patterns-python-cache
+  - "tools/requirements.json"
+  - "tools/requirements/requirements.*.txt"
+
 .patterns-python-files: &patterns-python-files
   - ".gitlab/ci/static-code-analysis.yml"
   - "**/*.py"
@@ -232,6 +236,11 @@
   rules:
     - <<: *if-protected-no_label
 
+.rules:patterns:python-cache:
+  rules:
+    - <<: *if-dev-push
+      changes: *patterns-python-cache
+
 .rules:dev:
   rules:
     - <<: *if-trigger

.gitlab/ci/upload_cache.yml

@@ -0,0 +1,36 @@
+.upload_cache_template:
+  stage: upload_cache
+  image: $ESP_ENV_IMAGE
+
+.upload_pip_cache_template:
+  extends:
+    - .upload_cache_template
+    - .rules:patterns:python-cache
+  cache:
+    key: pip-cache
+    paths:
+      - .cache/pip
+    # pull only for most of the use cases since it's cache dir.
+    # Only set "push" policy for "upload_cache" stage jobs since it would install all pypi packages
+    policy: push
+  before_script: []
+  script:
+    - source tools/ci/utils.sh
+    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
+    - source tools/ci/setup_python.sh
+    - rm -rf .cache/pip  # clear old packages
+    - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest
+    # TODO: remove this, IDFCI-1207
+    - pip install esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
+
+upload-pip-cache-shiny:
+  extends: .upload_pip_cache_template
+  tags:
+    - shiny
+    - build
+
+upload-pip-cache-brew:
+  extends: .upload_pip_cache_template
+  tags:
+    - brew
+    - build

docs/en/contribute/esp-idf-tests-with-pytest.rst

@@ -22,7 +22,7 @@ All the introduced concepts and usages are based on the default behavior in ESP-
 Installation
 ------------
 
-``$ pip install -U pytest-embedded-serial-esp~=0.7.0 pytest-embedded-idf~=0.7.0``
+All dependencies could be installed by running the install script with the ``--enable-pytest`` argument, e.g. ``$ install.sh --enable-pytest``.
 
 Basic Concepts
 --------------

pytest.ini

@@ -43,7 +43,8 @@ markers =
   ethernet: ethernet runner
   ethernet_flash_8m: ethernet runner with 8mb flash
   wifi: wifi runner
-  wifi_bt
+  wifi_bt: wifi runner with bluetooth
+  deepsleep: deepsleep runners
 
   # multi-dut markers
   multi_dut_generic: tests should be run on generic runners, at least have two duts connected.

tools/requirements.json

@@ -12,6 +12,12 @@
             "description": "Packages for supporting debugging from web browser",
             "optional": true,
             "requirement_path": "tools/requirements/requirements.gdbgui.txt"
+        },
+        {
+            "name": "pytest",
+            "description": "Packages for CI with pytest",
+            "optional": true,
+            "requirement_path": "tools/requirements/requirements.pytest.txt"
         }
     ]
 }

tools/requirements/requirements.pytest.txt

@@ -0,0 +1,10 @@
+pytest-embedded-serial-esp
+pytest-embedded-idf
+pytest-embedded-qemu
+pytest-rerunfailures
+scapy
+websocket-client
+netifaces
+rangehttpserver
+dbus-python; sys_platform != 'win32'
+protobuf