Merge branch 'feature/support_python3_ci' into 'master'

ci: All jobs use python3 by default

See merge request espressif/esp-idf!11154

examples/bluetooth/nimble/bleprph/bleprph_test.py

@@ -89,7 +89,7 @@ def bleprph_client_task(prph_obj, dut, dut_addr):
     - write 'A' to characteristic with write permission
     '''
     chars_ret_on_write = {}
-    chars_ret_on_write = ble_client_obj.write_chars('A')
+    chars_ret_on_write = ble_client_obj.write_chars(b'A')
     if chars_ret_on_write:
         Utility.console_log("\nCharacteristics after write operation")
         for path, props in chars_ret_on_write.items():

examples/protocols/http_server/simple/http_server_simple_test.py

@@ -100,12 +100,6 @@ def test_examples_protocol_http_server_simple(env, extra_data):
         raise RuntimeError
     dut1.expect("Found URL query => " + query, timeout=30)
 
-    query = "abcd\nyz"
-    Utility.console_log("Test /hello with invalid query")
-    if client.test_custom_uri_query(got_ip, got_port, query):
-        raise RuntimeError
-    dut1.expect("400 Bad Request - Server unable to understand request due to invalid syntax", timeout=30)
-
 
 if __name__ == '__main__':
     test_examples_protocol_http_server_simple()

examples/protocols/http_server/ws_echo_server/ws_server_example_test.py

@@ -84,6 +84,7 @@ def test_examples_protocol_http_ws_echo_server(env, extra_data):
             ws.write(data=DATA, opcode=expected_opcode)
             opcode, data = ws.read()
             Utility.console_log("Testing opcode {}: Received opcode:{}, data:{}".format(expected_opcode, opcode, data))
+            data = data.decode()
             if expected_opcode == OPCODE_PING:
                 dut1.expect("Got a WS PING frame, Replying PONG")
                 if opcode != OPCODE_PONG or data != DATA:
@@ -96,6 +97,7 @@ def test_examples_protocol_http_ws_echo_server(env, extra_data):
         ws.write(data="Trigger async", opcode=OPCODE_TEXT)
         opcode, data = ws.read()
         Utility.console_log("Testing async send: Received opcode:{}, data:{}".format(opcode, data))
+        data = data.decode()
         if opcode != OPCODE_TEXT or data != "Async data":
             raise RuntimeError("Failed to receive correct opcode:{} or data:{}".format(opcode, data))
 

examples/protocols/sockets/tcp_server/example_test.py

@@ -36,13 +36,13 @@ def tcp_client(address, payload):
         print('Could not open socket: ', msg)
         sock.close()
         raise
-    sock.sendall(payload)
+    sock.sendall(payload.encode())
     data = sock.recv(1024)
     if not data:
         return
     print('Reply : ' + data.decode())
     sock.close()
-    return data
+    return data.decode()
 
 
 @ttfw_idf.idf_example_test(env_tag="Example_WIFI")

examples/protocols/sockets/udp_server/example_test.py

@@ -31,7 +31,7 @@ def udp_client(address, payload):
         print('Could not create socket: ' + str(msg[0]) + ': ' + msg[1])
         raise
     try:
-        sock.sendto(payload, addr)
+        sock.sendto(payload.encode(), addr)
         reply, addr = sock.recvfrom(128)
         if not reply:
             return
@@ -40,7 +40,7 @@ def udp_client(address, payload):
         print('Error Code : ' + str(msg[0]) + ' Message: ' + msg[1])
         sock.close()
         raise
-    return reply
+    return reply.decode()
 
 
 @ttfw_idf.idf_example_test(env_tag="Example_WIFI")

examples/system/ota/advanced_https_ota/example_test.py

@@ -1,5 +1,6 @@
 import re
 import os
+import struct
 import socket
 import http.server
 from threading import Thread
@@ -334,12 +335,12 @@ def test_examples_protocol_advanced_https_ota_example_random(env, extra_data):
     random_bin_size = 32000
     # check and log bin size
     binary_file = os.path.join(dut1.app.binary_path, random_bin_name)
-    fo = open(binary_file, "w+")
+    fo = open(binary_file, "wb+")
     # First byte of binary file is always set to zero. If first byte is generated randomly,
     # in some cases it may generate 0xE9 which will result in failure of testcase.
-    fo.write(str(0))
+    fo.write(struct.pack("B", 0))
     for i in range(random_bin_size - 1):
-        fo.write(str(random.randrange(0,255,1)))
+        fo.write(struct.pack("B", random.randrange(0,255,1)))
     fo.close()
     bin_size = os.path.getsize(binary_file)
     ttfw_idf.log_performance("advanced_https_ota_bin_size", "{}KB".format(bin_size // 1024))

examples/system/ota/native_ota_example/example_test.py

@@ -1,5 +1,6 @@
 import re
 import os
+import struct
 import socket
 import http.server
 from threading import Thread
@@ -301,12 +302,12 @@ def test_examples_protocol_native_ota_example_random(env, extra_data):
     random_bin_size = 32000
     # check and log bin size
     binary_file = os.path.join(dut1.app.binary_path, random_bin_name)
-    fo = open(binary_file, "w+")
+    fo = open(binary_file, "wb+")
     # First byte of binary file is always set to zero. If first byte is generated randomly,
     # in some cases it may generate 0xE9 which will result in failure of testcase.
-    fo.write(str(0))
+    fo.write(struct.pack("B", 0))
     for i in range(random_bin_size - 1):
-        fo.write(str(random.randrange(0,255,1)))
+        fo.write(struct.pack("B", random.randrange(0,255,1)))
     fo.close()
     bin_size = os.path.getsize(binary_file)
     ttfw_idf.log_performance("native_ota_bin_size", "{}KB".format(bin_size // 1024))

tools/ci/config/assign-test.yml

@@ -22,6 +22,7 @@ assign_test:
     INTEGRATION_TEST_CASE_PATH: "${CI_PROJECT_DIR}/auto_test_script/TestCaseFiles"
     ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py"
     PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages
+    # auto_test_script only supports python 3.7.x
     PYTHON_VER: 3.7.7
   artifacts:
     paths:

tools/ci/config/host-test.yml

@@ -74,6 +74,8 @@ test_ldgen_on_host:
     - cd tools/ldgen/test
     - ./test_fragments.py
     - ./test_generation.py
+  variables:
+    LC_ALL: C.UTF-8
 
 test_mdns_fuzzer_on_host:
   extends: .host_fuzzer_test_template

tools/ci/config/target-test.yml

@@ -60,7 +60,6 @@
   variables:
     TEST_CASE_PATH: "$COMPONENT_UT_DIRS"
     CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/component_ut/test_configs"
-    PYTHON_VER: 3
   script:
     - *define_config_file_name
     # first test if config file exists, if not exist, exit 0
@@ -80,7 +79,6 @@
   variables:
     TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/unit-test-app"
     CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/unit_test/CIConfigs"
-    PYTHON_VER: 3
 
 .integration_test_template:
   extends:
@@ -98,6 +96,7 @@
     KNOWN_ISSUE_FILE: "${CI_PROJECT_DIR}/components/idf_test/integration_test/KnownIssues"
     CI_RUNNER_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIRunner.py"
     PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages
+    # auto_test_script only supports python 3.7.x
     PYTHON_VER: 3.7.7
   script:
     - *define_config_file_name

tools/ci/setup_python.sh

@@ -1,19 +1,10 @@
 #!/usr/bin/env bash
 
-# py2_incomp='assign_test|nvs_compatible_test|IT'
-# - auto_test_script do not support python2 anymore
-# but there are still some jobs incompatible with Python 3 (eg: example_test)
-# keep the default python interpreter as 2.7.15 until all the jobs support python3
-
-# Regexp for matching job names which are incompatible with Python 3
-# - UT_009_ - multi-device tests are not compatible
-# - UT_017_ - multi-device tests are not compatible
-py3_incomp='UT_009_|UT_017_'
-
-if [ -z ${PYTHON_VER+x} ] || [[ $CI_JOB_NAME =~ $py3_incomp ]]; then
-    # Use this version of the Python interpreter if it was not defined before or
-    # the given job is not compatible with Python 3
-    PYTHON_VER=2.7.15
+if [ -z ${PYTHON_VER+x} ]; then
+    # Use this version of the Python interpreter if it was not defined before.
+    # 3.4.8 is the default python3 interpreter in esp32-ci-env
+    # Jobs which doesn't support this version should define PYTHON_VER themselves
+    PYTHON_VER=3.4.8
 fi
 
 if [ -f /opt/pyenv/activate ];

tools/esp_prov/__init__.py

@@ -1 +1 @@
-from esp_prov import *  # noqa: export esp_prov module to users
+from .esp_prov import *  # noqa: export esp_prov module to users

tools/esp_prov/esp_prov.py

@@ -16,7 +16,7 @@
 #
 
 from __future__ import print_function
-from builtins import input
+from builtins import input as binput
 import argparse
 import textwrap
 import time
@@ -471,7 +471,7 @@ if __name__ == '__main__':
 
             while True:
                 try:
-                    select = int(input("Select AP by number (0 to rescan) : "))
+                    select = int(binput("Select AP by number (0 to rescan) : "))
                     if select < 0 or select > len(APs):
                         raise ValueError
                     break

tools/esp_prov/prov/custom_prov.py

@@ -48,7 +48,7 @@ def custom_config_response(security_ctx, response_data):
 
 def custom_data_request(security_ctx, data):
     # Encrypt the custom data
-    enc_cmd = security_ctx.encrypt_data(data)
+    enc_cmd = security_ctx.encrypt_data(tobytes(data))
     print_verbose(security_ctx, "Client -> Device (CustomData cmd) " + utils.str_to_hexstr(enc_cmd))
     return enc_cmd
 

tools/esp_prov/security/security1.py

@@ -167,7 +167,7 @@ class Security1(Security):
             return -1
 
     def encrypt_data(self, data):
-        return self.cipher.update(data)
+        return self.cipher.update(tobytes(data))
 
     def decrypt_data(self, data):
-        return self.cipher.update(data)
+        return self.cipher.update(tobytes(data))

tools/esp_prov/transport/ble_cli.py

@@ -234,6 +234,8 @@ class BLE_Bluez_Client:
 
         try:
             path.WriteValue([ord(c) for c in data], {}, dbus_interface='org.bluez.GattCharacteristic1')
+        except TypeError:  # python3 compatible
+            path.WriteValue([c for c in data], {}, dbus_interface='org.bluez.GattCharacteristic1')
         except dbus.exceptions.DBusException as e:
             raise RuntimeError("Failed to write value to characteristic " + characteristic_uuid + ": " + str(e))
 

tools/esp_prov/utils/convenience.py

@@ -14,12 +14,14 @@
 #
 
 # Convenience functions for commonly used data type conversions
+import binascii
+from future.utils import tobytes
 
 
 def str_to_hexstr(string):
     # Form hexstr by appending ASCII codes (in hex) corresponding to
     # each character in the input string
-    return ''.join('{:02x}'.format(ord(c)) for c in string)
+    return binascii.hexlify(tobytes(string)).decode()
 
 
 def hexstr_to_str(hexstr):
@@ -28,4 +30,4 @@ def hexstr_to_str(hexstr):
         hexstr = '0' + hexstr
     # Interpret consecutive pairs of hex characters as 8 bit ASCII codes
     # and append characters corresponding to each code to form the string
-    return ''.join(chr(int(hexstr[2 * i: 2 * i + 2], 16)) for i in range(len(hexstr) // 2))
+    return binascii.unhexlify(tobytes(hexstr)).decode()