From 454edcc0a876acf7e4f4cac9db8726b16384df4c Mon Sep 17 00:00:00 2001 From: Chen Yu Dong Date: Wed, 11 Nov 2020 11:31:17 +0800 Subject: [PATCH] ci: all jobs use python3 by default --- .../bluetooth/nimble/bleprph/bleprph_test.py | 2 +- .../simple/http_server_simple_test.py | 6 ------ .../ws_echo_server/ws_server_example_test.py | 2 ++ .../sockets/tcp_server/example_test.py | 4 ++-- .../sockets/udp_server/example_test.py | 4 ++-- .../ota/advanced_https_ota/example_test.py | 7 ++++--- .../ota/native_ota_example/example_test.py | 7 ++++--- tools/ci/config/assign-test.yml | 1 + tools/ci/config/host-test.yml | 2 ++ tools/ci/config/target-test.yml | 3 +-- tools/ci/setup_python.sh | 19 +++++-------------- tools/esp_prov/__init__.py | 2 +- tools/esp_prov/esp_prov.py | 4 ++-- tools/esp_prov/prov/custom_prov.py | 2 +- tools/esp_prov/security/security1.py | 4 ++-- tools/esp_prov/transport/ble_cli.py | 2 ++ tools/esp_prov/utils/convenience.py | 6 ++++-- 17 files changed, 36 insertions(+), 41 deletions(-) diff --git a/examples/bluetooth/nimble/bleprph/bleprph_test.py b/examples/bluetooth/nimble/bleprph/bleprph_test.py index 8a6495101c..80a5c947ac 100644 --- a/examples/bluetooth/nimble/bleprph/bleprph_test.py +++ b/examples/bluetooth/nimble/bleprph/bleprph_test.py @@ -89,7 +89,7 @@ def bleprph_client_task(prph_obj, dut, dut_addr): - write 'A' to characteristic with write permission ''' chars_ret_on_write = {} - chars_ret_on_write = ble_client_obj.write_chars('A') + chars_ret_on_write = ble_client_obj.write_chars(b'A') if chars_ret_on_write: Utility.console_log("\nCharacteristics after write operation") for path, props in chars_ret_on_write.items(): diff --git a/examples/protocols/http_server/simple/http_server_simple_test.py b/examples/protocols/http_server/simple/http_server_simple_test.py index 54c894ea7e..14759e5db0 100644 --- a/examples/protocols/http_server/simple/http_server_simple_test.py +++ b/examples/protocols/http_server/simple/http_server_simple_test.py @@ -100,12 +100,6 @@ def test_examples_protocol_http_server_simple(env, extra_data): raise RuntimeError dut1.expect("Found URL query => " + query, timeout=30) - query = "abcd\nyz" - Utility.console_log("Test /hello with invalid query") - if client.test_custom_uri_query(got_ip, got_port, query): - raise RuntimeError - dut1.expect("400 Bad Request - Server unable to understand request due to invalid syntax", timeout=30) - if __name__ == '__main__': test_examples_protocol_http_server_simple() diff --git a/examples/protocols/http_server/ws_echo_server/ws_server_example_test.py b/examples/protocols/http_server/ws_echo_server/ws_server_example_test.py index 2d53b698f0..bf2165ae76 100644 --- a/examples/protocols/http_server/ws_echo_server/ws_server_example_test.py +++ b/examples/protocols/http_server/ws_echo_server/ws_server_example_test.py @@ -84,6 +84,7 @@ def test_examples_protocol_http_ws_echo_server(env, extra_data): ws.write(data=DATA, opcode=expected_opcode) opcode, data = ws.read() Utility.console_log("Testing opcode {}: Received opcode:{}, data:{}".format(expected_opcode, opcode, data)) + data = data.decode() if expected_opcode == OPCODE_PING: dut1.expect("Got a WS PING frame, Replying PONG") if opcode != OPCODE_PONG or data != DATA: @@ -96,6 +97,7 @@ def test_examples_protocol_http_ws_echo_server(env, extra_data): ws.write(data="Trigger async", opcode=OPCODE_TEXT) opcode, data = ws.read() Utility.console_log("Testing async send: Received opcode:{}, data:{}".format(opcode, data)) + data = data.decode() if opcode != OPCODE_TEXT or data != "Async data": raise RuntimeError("Failed to receive correct opcode:{} or data:{}".format(opcode, data)) diff --git a/examples/protocols/sockets/tcp_server/example_test.py b/examples/protocols/sockets/tcp_server/example_test.py index 11b631bd92..d28273f3a1 100644 --- a/examples/protocols/sockets/tcp_server/example_test.py +++ b/examples/protocols/sockets/tcp_server/example_test.py @@ -36,13 +36,13 @@ def tcp_client(address, payload): print('Could not open socket: ', msg) sock.close() raise - sock.sendall(payload) + sock.sendall(payload.encode()) data = sock.recv(1024) if not data: return print('Reply : ' + data.decode()) sock.close() - return data + return data.decode() @ttfw_idf.idf_example_test(env_tag="Example_WIFI") diff --git a/examples/protocols/sockets/udp_server/example_test.py b/examples/protocols/sockets/udp_server/example_test.py index 0173c8c37e..fa4f9d60d3 100644 --- a/examples/protocols/sockets/udp_server/example_test.py +++ b/examples/protocols/sockets/udp_server/example_test.py @@ -31,7 +31,7 @@ def udp_client(address, payload): print('Could not create socket: ' + str(msg[0]) + ': ' + msg[1]) raise try: - sock.sendto(payload, addr) + sock.sendto(payload.encode(), addr) reply, addr = sock.recvfrom(128) if not reply: return @@ -40,7 +40,7 @@ def udp_client(address, payload): print('Error Code : ' + str(msg[0]) + ' Message: ' + msg[1]) sock.close() raise - return reply + return reply.decode() @ttfw_idf.idf_example_test(env_tag="Example_WIFI") diff --git a/examples/system/ota/advanced_https_ota/example_test.py b/examples/system/ota/advanced_https_ota/example_test.py index 3b2d64371f..da1bdf861e 100644 --- a/examples/system/ota/advanced_https_ota/example_test.py +++ b/examples/system/ota/advanced_https_ota/example_test.py @@ -1,5 +1,6 @@ import re import os +import struct import socket import http.server from threading import Thread @@ -334,12 +335,12 @@ def test_examples_protocol_advanced_https_ota_example_random(env, extra_data): random_bin_size = 32000 # check and log bin size binary_file = os.path.join(dut1.app.binary_path, random_bin_name) - fo = open(binary_file, "w+") + fo = open(binary_file, "wb+") # First byte of binary file is always set to zero. If first byte is generated randomly, # in some cases it may generate 0xE9 which will result in failure of testcase. - fo.write(str(0)) + fo.write(struct.pack("B", 0)) for i in range(random_bin_size - 1): - fo.write(str(random.randrange(0,255,1))) + fo.write(struct.pack("B", random.randrange(0,255,1))) fo.close() bin_size = os.path.getsize(binary_file) ttfw_idf.log_performance("advanced_https_ota_bin_size", "{}KB".format(bin_size // 1024)) diff --git a/examples/system/ota/native_ota_example/example_test.py b/examples/system/ota/native_ota_example/example_test.py index fbfd980685..b105e319be 100644 --- a/examples/system/ota/native_ota_example/example_test.py +++ b/examples/system/ota/native_ota_example/example_test.py @@ -1,5 +1,6 @@ import re import os +import struct import socket import http.server from threading import Thread @@ -301,12 +302,12 @@ def test_examples_protocol_native_ota_example_random(env, extra_data): random_bin_size = 32000 # check and log bin size binary_file = os.path.join(dut1.app.binary_path, random_bin_name) - fo = open(binary_file, "w+") + fo = open(binary_file, "wb+") # First byte of binary file is always set to zero. If first byte is generated randomly, # in some cases it may generate 0xE9 which will result in failure of testcase. - fo.write(str(0)) + fo.write(struct.pack("B", 0)) for i in range(random_bin_size - 1): - fo.write(str(random.randrange(0,255,1))) + fo.write(struct.pack("B", random.randrange(0,255,1))) fo.close() bin_size = os.path.getsize(binary_file) ttfw_idf.log_performance("native_ota_bin_size", "{}KB".format(bin_size // 1024)) diff --git a/tools/ci/config/assign-test.yml b/tools/ci/config/assign-test.yml index 41e45e3934..02d0c8ce83 100644 --- a/tools/ci/config/assign-test.yml +++ b/tools/ci/config/assign-test.yml @@ -22,6 +22,7 @@ assign_test: INTEGRATION_TEST_CASE_PATH: "${CI_PROJECT_DIR}/auto_test_script/TestCaseFiles" ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py" PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages + # auto_test_script only supports python 3.7.x PYTHON_VER: 3.7.7 artifacts: paths: diff --git a/tools/ci/config/host-test.yml b/tools/ci/config/host-test.yml index b1a19020cf..d5f3b6b219 100644 --- a/tools/ci/config/host-test.yml +++ b/tools/ci/config/host-test.yml @@ -74,6 +74,8 @@ test_ldgen_on_host: - cd tools/ldgen/test - ./test_fragments.py - ./test_generation.py + variables: + LC_ALL: C.UTF-8 test_mdns_fuzzer_on_host: extends: .host_fuzzer_test_template diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index e6891464e0..c5f17599a2 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -60,7 +60,6 @@ variables: TEST_CASE_PATH: "$COMPONENT_UT_DIRS" CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/component_ut/test_configs" - PYTHON_VER: 3 script: - *define_config_file_name # first test if config file exists, if not exist, exit 0 @@ -80,7 +79,6 @@ variables: TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/unit-test-app" CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/unit_test/CIConfigs" - PYTHON_VER: 3 .integration_test_template: extends: @@ -98,6 +96,7 @@ KNOWN_ISSUE_FILE: "${CI_PROJECT_DIR}/components/idf_test/integration_test/KnownIssues" CI_RUNNER_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIRunner.py" PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages + # auto_test_script only supports python 3.7.x PYTHON_VER: 3.7.7 script: - *define_config_file_name diff --git a/tools/ci/setup_python.sh b/tools/ci/setup_python.sh index c23e58ce54..6cc8fb78ba 100644 --- a/tools/ci/setup_python.sh +++ b/tools/ci/setup_python.sh @@ -1,19 +1,10 @@ #!/usr/bin/env bash -# py2_incomp='assign_test|nvs_compatible_test|IT' -# - auto_test_script do not support python2 anymore -# but there are still some jobs incompatible with Python 3 (eg: example_test) -# keep the default python interpreter as 2.7.15 until all the jobs support python3 - -# Regexp for matching job names which are incompatible with Python 3 -# - UT_009_ - multi-device tests are not compatible -# - UT_017_ - multi-device tests are not compatible -py3_incomp='UT_009_|UT_017_' - -if [ -z ${PYTHON_VER+x} ] || [[ $CI_JOB_NAME =~ $py3_incomp ]]; then - # Use this version of the Python interpreter if it was not defined before or - # the given job is not compatible with Python 3 - PYTHON_VER=2.7.15 +if [ -z ${PYTHON_VER+x} ]; then + # Use this version of the Python interpreter if it was not defined before. + # 3.4.8 is the default python3 interpreter in esp32-ci-env + # Jobs which doesn't support this version should define PYTHON_VER themselves + PYTHON_VER=3.4.8 fi if [ -f /opt/pyenv/activate ]; diff --git a/tools/esp_prov/__init__.py b/tools/esp_prov/__init__.py index cbd5e5b60f..d0a872a2c3 100644 --- a/tools/esp_prov/__init__.py +++ b/tools/esp_prov/__init__.py @@ -1 +1 @@ -from esp_prov import * # noqa: export esp_prov module to users +from .esp_prov import * # noqa: export esp_prov module to users diff --git a/tools/esp_prov/esp_prov.py b/tools/esp_prov/esp_prov.py index 7f010522b5..8215a10172 100644 --- a/tools/esp_prov/esp_prov.py +++ b/tools/esp_prov/esp_prov.py @@ -16,7 +16,7 @@ # from __future__ import print_function -from builtins import input +from builtins import input as binput import argparse import textwrap import time @@ -471,7 +471,7 @@ if __name__ == '__main__': while True: try: - select = int(input("Select AP by number (0 to rescan) : ")) + select = int(binput("Select AP by number (0 to rescan) : ")) if select < 0 or select > len(APs): raise ValueError break diff --git a/tools/esp_prov/prov/custom_prov.py b/tools/esp_prov/prov/custom_prov.py index 06ec1ca717..87b499e6e1 100644 --- a/tools/esp_prov/prov/custom_prov.py +++ b/tools/esp_prov/prov/custom_prov.py @@ -48,7 +48,7 @@ def custom_config_response(security_ctx, response_data): def custom_data_request(security_ctx, data): # Encrypt the custom data - enc_cmd = security_ctx.encrypt_data(data) + enc_cmd = security_ctx.encrypt_data(tobytes(data)) print_verbose(security_ctx, "Client -> Device (CustomData cmd) " + utils.str_to_hexstr(enc_cmd)) return enc_cmd diff --git a/tools/esp_prov/security/security1.py b/tools/esp_prov/security/security1.py index 90bffff8a4..be657186c3 100644 --- a/tools/esp_prov/security/security1.py +++ b/tools/esp_prov/security/security1.py @@ -167,7 +167,7 @@ class Security1(Security): return -1 def encrypt_data(self, data): - return self.cipher.update(data) + return self.cipher.update(tobytes(data)) def decrypt_data(self, data): - return self.cipher.update(data) + return self.cipher.update(tobytes(data)) diff --git a/tools/esp_prov/transport/ble_cli.py b/tools/esp_prov/transport/ble_cli.py index ad80bf2ddc..5d88ece806 100644 --- a/tools/esp_prov/transport/ble_cli.py +++ b/tools/esp_prov/transport/ble_cli.py @@ -234,6 +234,8 @@ class BLE_Bluez_Client: try: path.WriteValue([ord(c) for c in data], {}, dbus_interface='org.bluez.GattCharacteristic1') + except TypeError: # python3 compatible + path.WriteValue([c for c in data], {}, dbus_interface='org.bluez.GattCharacteristic1') except dbus.exceptions.DBusException as e: raise RuntimeError("Failed to write value to characteristic " + characteristic_uuid + ": " + str(e)) diff --git a/tools/esp_prov/utils/convenience.py b/tools/esp_prov/utils/convenience.py index 9fec1deac6..42d0febe26 100644 --- a/tools/esp_prov/utils/convenience.py +++ b/tools/esp_prov/utils/convenience.py @@ -14,12 +14,14 @@ # # Convenience functions for commonly used data type conversions +import binascii +from future.utils import tobytes def str_to_hexstr(string): # Form hexstr by appending ASCII codes (in hex) corresponding to # each character in the input string - return ''.join('{:02x}'.format(ord(c)) for c in string) + return binascii.hexlify(tobytes(string)).decode() def hexstr_to_str(hexstr): @@ -28,4 +30,4 @@ def hexstr_to_str(hexstr): hexstr = '0' + hexstr # Interpret consecutive pairs of hex characters as 8 bit ASCII codes # and append characters corresponding to each code to form the string - return ''.join(chr(int(hexstr[2 * i: 2 * i + 2], 16)) for i in range(len(hexstr) // 2)) + return binascii.unhexlify(tobytes(hexstr)).decode()