From bfd6ad94ff950a4b7e3a2125db1539c5e4ca333a Mon Sep 17 00:00:00 2001 From: Carlosgg Date: Thu, 30 Nov 2023 16:44:48 +0000 Subject: [PATCH] extmod/asyncio: Add ssl support with SSLContext. This adds asyncio ssl support with SSLContext and the corresponding tests in `tests/net_inet` and `tests/multi_net`. Note that not doing the handshake on connect will delegate the handshake to the following `mbedtls_ssl_read/write` calls. However if the handshake fails when a client certificate is required and not presented by the peer, it needs to be notified of this handshake error (otherwise it will hang until timeout if any). Finally at MicroPython side raise the proper mbedtls error code and message. Signed-off-by: Carlos Gil --- .gitattributes | 1 + extmod/asyncio/stream.py | 29 +++++-- extmod/modssl_mbedtls.c | 42 ++++++++++ tests/multi_net/asyncio_tls_server_client.py | 73 +++++++++++++++++ .../asyncio_tls_server_client.py.exp | 8 ++ ...o_tls_server_client_cert_required_error.py | 76 +++++++++++++++++ ...s_server_client_cert_required_error.py.exp | 10 +++ .../asyncio_tls_server_client_readline.py | 77 ++++++++++++++++++ .../asyncio_tls_server_client_readline.py.exp | 10 +++ .../asyncio_tls_server_client_verify_error.py | 77 ++++++++++++++++++ ...ncio_tls_server_client_verify_error.py.exp | 12 +++ .../asyncio_tls_open_connection_readline.py | 59 ++++++++++++++ tests/net_inet/isrg.der | Bin 0 -> 1391 bytes 13 files changed, 469 insertions(+), 5 deletions(-) create mode 100644 tests/multi_net/asyncio_tls_server_client.py create mode 100644 tests/multi_net/asyncio_tls_server_client.py.exp create mode 100644 tests/multi_net/asyncio_tls_server_client_cert_required_error.py create mode 100644 tests/multi_net/asyncio_tls_server_client_cert_required_error.py.exp create mode 100644 tests/multi_net/asyncio_tls_server_client_readline.py create mode 100644 tests/multi_net/asyncio_tls_server_client_readline.py.exp create mode 100644 tests/multi_net/asyncio_tls_server_client_verify_error.py create mode 100644 tests/multi_net/asyncio_tls_server_client_verify_error.py.exp create mode 100644 tests/net_inet/asyncio_tls_open_connection_readline.py create mode 100644 tests/net_inet/isrg.der diff --git a/.gitattributes b/.gitattributes index fe0c4b47e0..2d8496db50 100644 --- a/.gitattributes +++ b/.gitattributes @@ -13,6 +13,7 @@ *.jpg binary *.dxf binary *.mpy binary +*.der binary # These should also not be modified by git. tests/basics/string_cr_conversion.py -text diff --git a/extmod/asyncio/stream.py b/extmod/asyncio/stream.py index 5547bfbd51..bcc2a13a86 100644 --- a/extmod/asyncio/stream.py +++ b/extmod/asyncio/stream.py @@ -63,6 +63,8 @@ class Stream: while True: yield core._io_queue.queue_read(self.s) l2 = self.s.readline() # may do multiple reads but won't block + if l2 is None: + continue l += l2 if not l2 or l[-1] == 10: # \n (check l in case l2 is str) return l @@ -100,19 +102,29 @@ StreamWriter = Stream # Create a TCP stream connection to a remote host # # async -def open_connection(host, port): +def open_connection(host, port, ssl=None, server_hostname=None): from errno import EINPROGRESS import socket ai = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)[0] # TODO this is blocking! s = socket.socket(ai[0], ai[1], ai[2]) s.setblocking(False) - ss = Stream(s) try: s.connect(ai[-1]) except OSError as er: if er.errno != EINPROGRESS: raise er + # wrap with SSL, if requested + if ssl: + if ssl is True: + import ssl as _ssl + + ssl = _ssl.SSLContext(_ssl.PROTOCOL_TLS_CLIENT) + if not server_hostname: + server_hostname = host + s = ssl.wrap_socket(s, server_hostname=server_hostname, do_handshake_on_connect=False) + s.setblocking(False) + ss = Stream(s) yield core._io_queue.queue_write(s) return ss, ss @@ -135,7 +147,7 @@ class Server: async def wait_closed(self): await self.task - async def _serve(self, s, cb): + async def _serve(self, s, cb, ssl): self.state = False # Accept incoming connections while True: @@ -156,6 +168,13 @@ class Server: except: # Ignore a failed accept continue + if ssl: + try: + s2 = ssl.wrap_socket(s2, server_side=True, do_handshake_on_connect=False) + except OSError as e: + core.sys.print_exception(e) + s2.close() + continue s2.setblocking(False) s2s = Stream(s2, {"peername": addr}) core.create_task(cb(s2s, s2s)) @@ -163,7 +182,7 @@ class Server: # Helper function to start a TCP stream server, running as a new task # TODO could use an accept-callback on socket read activity instead of creating a task -async def start_server(cb, host, port, backlog=5): +async def start_server(cb, host, port, backlog=5, ssl=None): import socket # Create and bind server socket. @@ -176,7 +195,7 @@ async def start_server(cb, host, port, backlog=5): # Create and return server object and task. srv = Server() - srv.task = core.create_task(srv._serve(s, cb)) + srv.task = core.create_task(srv._serve(s, cb, ssl)) try: # Ensure that the _serve task has been scheduled so that it gets to # handle cancellation. diff --git a/extmod/modssl_mbedtls.c b/extmod/modssl_mbedtls.c index f407d94cbf..0190c96a9a 100644 --- a/extmod/modssl_mbedtls.c +++ b/extmod/modssl_mbedtls.c @@ -166,6 +166,46 @@ STATIC NORETURN void mbedtls_raise_error(int err) { #endif } +STATIC void ssl_check_async_handshake_failure(mp_obj_ssl_socket_t *sslsock, int *errcode) { + if ( + #if MBEDTLS_VERSION_NUMBER >= 0x03000000 + (*errcode < 0) && (mbedtls_ssl_is_handshake_over(&sslsock->ssl) == 0) && (*errcode != MBEDTLS_ERR_SSL_CONN_EOF) + #else + (*errcode < 0) && (*errcode != MBEDTLS_ERR_SSL_CONN_EOF) + #endif + ) { + // Asynchronous handshake is done by mbdetls_ssl_read/write. If the return code is + // MBEDTLS_ERR_XX (i.e < 0) and the handshake is not done due to a handshake failure, + // then notify peer with proper error code and raise local error with mbedtls_raise_error. + + if (*errcode == MBEDTLS_ERR_SSL_NO_CLIENT_CERTIFICATE) { + // Check if TLSv1.3 and use proper alert for this case (to be implemented) + // uint8_t alert = MBEDTLS_SSL_ALERT_MSG_CERT_REQUIRED; tlsv1.3 + // uint8_t alert = MBEDTLS_SSL_ALERT_MSG_HANDSHAKE_FAILURE; tlsv1.2 + mbedtls_ssl_send_alert_message(&sslsock->ssl, MBEDTLS_SSL_ALERT_LEVEL_FATAL, + MBEDTLS_SSL_ALERT_MSG_HANDSHAKE_FAILURE); + } + + if (*errcode == MBEDTLS_ERR_X509_CERT_VERIFY_FAILED) { + // The certificate may have been rejected for several reasons. + char xcbuf[256]; + uint32_t flags = mbedtls_ssl_get_verify_result(&sslsock->ssl); + int ret = mbedtls_x509_crt_verify_info(xcbuf, sizeof(xcbuf), "\n", flags); + // The length of the string written (not including the terminated nul byte), + // or a negative err code. + if (ret > 0) { + sslsock->sock = MP_OBJ_NULL; + mbedtls_ssl_free(&sslsock->ssl); + mp_raise_msg_varg(&mp_type_ValueError, MP_ERROR_TEXT("%s"), xcbuf); + } + } + + sslsock->sock = MP_OBJ_NULL; + mbedtls_ssl_free(&sslsock->ssl); + mbedtls_raise_error(*errcode); + } +} + /******************************************************************************/ // SSLContext type. @@ -614,6 +654,7 @@ STATIC mp_uint_t socket_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errc } else { o->last_error = ret; } + ssl_check_async_handshake_failure(o, &ret); *errcode = ret; return MP_STREAM_ERROR; } @@ -642,6 +683,7 @@ STATIC mp_uint_t socket_write(mp_obj_t o_in, const void *buf, mp_uint_t size, in } else { o->last_error = ret; } + ssl_check_async_handshake_failure(o, &ret); *errcode = ret; return MP_STREAM_ERROR; } diff --git a/tests/multi_net/asyncio_tls_server_client.py b/tests/multi_net/asyncio_tls_server_client.py new file mode 100644 index 0000000000..996cdb3e0d --- /dev/null +++ b/tests/multi_net/asyncio_tls_server_client.py @@ -0,0 +1,73 @@ +# Test asyncio TCP server and client with TLS, transferring some data. + +try: + import os + import asyncio + import ssl +except ImportError: + print("SKIP") + raise SystemExit + +PORT = 8000 + +# These are test certificates. See tests/README.md for details. +cert = cafile = "multi_net/rsa_cert.der" +key = "multi_net/rsa_key.der" + +try: + os.stat(cafile) + os.stat(key) +except OSError: + print("SKIP") + raise SystemExit + + +async def handle_connection(reader, writer): + data = await reader.read(100) + print("echo:", data) + writer.write(data) + await writer.drain() + + print("close") + writer.close() + await writer.wait_closed() + + print("done") + ev.set() + + +async def tcp_server(): + global ev + + server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_ctx.load_cert_chain(cert, key) + ev = asyncio.Event() + server = await asyncio.start_server(handle_connection, "0.0.0.0", PORT, ssl=server_ctx) + print("server running") + multitest.next() + async with server: + await asyncio.wait_for(ev.wait(), 10) + + +async def tcp_client(message): + client_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_ctx.verify_mode = ssl.CERT_REQUIRED + client_ctx.load_verify_locations(cafile=cafile) + reader, writer = await asyncio.open_connection( + IP, PORT, ssl=client_ctx, server_hostname="micropython.local" + ) + print("write:", message) + writer.write(message) + await writer.drain() + data = await reader.read(100) + print("read:", data) + + +def instance0(): + multitest.globals(IP=multitest.get_network_ip()) + asyncio.run(tcp_server()) + + +def instance1(): + multitest.next() + asyncio.run(tcp_client(b"client data")) diff --git a/tests/multi_net/asyncio_tls_server_client.py.exp b/tests/multi_net/asyncio_tls_server_client.py.exp new file mode 100644 index 0000000000..6dc6a9bbc7 --- /dev/null +++ b/tests/multi_net/asyncio_tls_server_client.py.exp @@ -0,0 +1,8 @@ +--- instance0 --- +server running +echo: b'client data' +close +done +--- instance1 --- +write: b'client data' +read: b'client data' diff --git a/tests/multi_net/asyncio_tls_server_client_cert_required_error.py b/tests/multi_net/asyncio_tls_server_client_cert_required_error.py new file mode 100644 index 0000000000..bd4d7b82ea --- /dev/null +++ b/tests/multi_net/asyncio_tls_server_client_cert_required_error.py @@ -0,0 +1,76 @@ +# Test asyncio TCP server and client with TLS, giving a cert required error. + +try: + import os + import asyncio + import ssl +except ImportError: + print("SKIP") + raise SystemExit + +PORT = 8000 + +# These are test certificates. See tests/README.md for details. +cert = cafile = "multi_net/rsa_cert.der" +key = "multi_net/rsa_key.der" + +try: + os.stat(cafile) + os.stat(key) +except OSError: + print("SKIP") + raise SystemExit + + +async def handle_connection(reader, writer): + print("handle connection") + try: + data = await reader.read(100) + except Exception as e: + print(e) + ev.set() + + +async def tcp_server(): + global ev + + server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_ctx.load_cert_chain(cert, key) + server_ctx.verify_mode = ssl.CERT_REQUIRED + server_ctx.load_verify_locations(cafile=cert) + ev = asyncio.Event() + server = await asyncio.start_server(handle_connection, "0.0.0.0", PORT, ssl=server_ctx) + print("server running") + multitest.next() + async with server: + await asyncio.wait_for(ev.wait(), 10) + multitest.wait("finished") + print("server done") + + +async def tcp_client(message): + client_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_ctx.verify_mode = ssl.CERT_REQUIRED + client_ctx.load_verify_locations(cafile=cafile) + reader, writer = await asyncio.open_connection( + IP, PORT, ssl=client_ctx, server_hostname="micropython.local" + ) + try: + print("write:", message) + writer.write(message) + print("drain") + await writer.drain() + except Exception as e: + print(e) + print("client done") + multitest.broadcast("finished") + + +def instance0(): + multitest.globals(IP=multitest.get_network_ip()) + asyncio.run(tcp_server()) + + +def instance1(): + multitest.next() + asyncio.run(tcp_client(b"client data")) diff --git a/tests/multi_net/asyncio_tls_server_client_cert_required_error.py.exp b/tests/multi_net/asyncio_tls_server_client_cert_required_error.py.exp new file mode 100644 index 0000000000..0f905d0d23 --- /dev/null +++ b/tests/multi_net/asyncio_tls_server_client_cert_required_error.py.exp @@ -0,0 +1,10 @@ +--- instance0 --- +server running +handle connection +(-29824, 'MBEDTLS_ERR_SSL_NO_CLIENT_CERTIFICATE') +server done +--- instance1 --- +write: b'client data' +drain +(-30592, 'MBEDTLS_ERR_SSL_FATAL_ALERT_MESSAGE') +client done diff --git a/tests/multi_net/asyncio_tls_server_client_readline.py b/tests/multi_net/asyncio_tls_server_client_readline.py new file mode 100644 index 0000000000..28add38f5d --- /dev/null +++ b/tests/multi_net/asyncio_tls_server_client_readline.py @@ -0,0 +1,77 @@ +# Test asyncio TCP server and client with TLS, using readline() to read data. + +try: + import os + import asyncio + import ssl +except ImportError: + print("SKIP") + raise SystemExit + +PORT = 8000 + +# These are test certificates. See tests/README.md for details. +cert = cafile = "multi_net/rsa_cert.der" +key = "multi_net/rsa_key.der" + +try: + os.stat(cafile) + os.stat(key) +except OSError: + print("SKIP") + raise SystemExit + + +async def handle_connection(reader, writer): + data = await reader.readline() + print("echo:", data) + data2 = await reader.readline() + print("echo:", data2) + writer.write(data + data2) + await writer.drain() + + print("close") + writer.close() + await writer.wait_closed() + + print("done") + ev.set() + + +async def tcp_server(): + global ev + + server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_ctx.load_cert_chain(cert, key) + ev = asyncio.Event() + server = await asyncio.start_server(handle_connection, "0.0.0.0", PORT, ssl=server_ctx) + print("server running") + multitest.next() + async with server: + await asyncio.wait_for(ev.wait(), 10) + + +async def tcp_client(message): + client_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_ctx.verify_mode = ssl.CERT_REQUIRED + client_ctx.load_verify_locations(cafile=cafile) + reader, writer = await asyncio.open_connection( + IP, PORT, ssl=client_ctx, server_hostname="micropython.local" + ) + print("write:", message) + writer.write(message) + await writer.drain() + data = await reader.readline() + print("read:", data) + data2 = await reader.readline() + print("read:", data2) + + +def instance0(): + multitest.globals(IP=multitest.get_network_ip()) + asyncio.run(tcp_server()) + + +def instance1(): + multitest.next() + asyncio.run(tcp_client(b"client data\nclient data2\n")) diff --git a/tests/multi_net/asyncio_tls_server_client_readline.py.exp b/tests/multi_net/asyncio_tls_server_client_readline.py.exp new file mode 100644 index 0000000000..4c93c5729a --- /dev/null +++ b/tests/multi_net/asyncio_tls_server_client_readline.py.exp @@ -0,0 +1,10 @@ +--- instance0 --- +server running +echo: b'client data\n' +echo: b'client data2\n' +close +done +--- instance1 --- +write: b'client data\nclient data2\n' +read: b'client data\n' +read: b'client data2\n' diff --git a/tests/multi_net/asyncio_tls_server_client_verify_error.py b/tests/multi_net/asyncio_tls_server_client_verify_error.py new file mode 100644 index 0000000000..46a476addf --- /dev/null +++ b/tests/multi_net/asyncio_tls_server_client_verify_error.py @@ -0,0 +1,77 @@ +# Test asyncio TCP server and client with TLS, and an incorrect server_hostname. + +try: + import os + import asyncio + import ssl +except ImportError: + print("SKIP") + raise SystemExit + +PORT = 8000 + +# These are test certificates. See tests/README.md for details. +cert = cafile = "multi_net/rsa_cert.der" +key = "multi_net/rsa_key.der" + +try: + os.stat(cafile) + os.stat(key) +except OSError: + print("SKIP") + raise SystemExit + + +async def handle_connection(reader, writer): + print("handle connection") + try: + data = await reader.read(100) + except Exception as e: + print(e) + ev.set() + + +async def tcp_server(): + global ev + + server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_ctx.load_cert_chain(cert, key) + ev = asyncio.Event() + server = await asyncio.start_server(handle_connection, "0.0.0.0", PORT, ssl=server_ctx) + print("server running") + multitest.next() + async with server: + await asyncio.wait_for(ev.wait(), 10) + print("server done") + multitest.broadcast("finished") + + +async def tcp_client(message): + client_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_ctx.verify_mode = ssl.CERT_REQUIRED + client_ctx.load_verify_locations(cafile=cafile) + reader, writer = await asyncio.open_connection( + IP, + PORT, + ssl=client_ctx, + server_hostname="foobar.local", # incorrect server_hostname + ) + try: + print("write:", message) + writer.write(message) + print("drain") + await writer.drain() + except Exception as e: + print(e) + multitest.wait("finished") + print("client done") + + +def instance0(): + multitest.globals(IP=multitest.get_network_ip()) + asyncio.run(tcp_server()) + + +def instance1(): + multitest.next() + asyncio.run(tcp_client(b"client data")) diff --git a/tests/multi_net/asyncio_tls_server_client_verify_error.py.exp b/tests/multi_net/asyncio_tls_server_client_verify_error.py.exp new file mode 100644 index 0000000000..36d0ab00f4 --- /dev/null +++ b/tests/multi_net/asyncio_tls_server_client_verify_error.py.exp @@ -0,0 +1,12 @@ +--- instance0 --- +server running +handle connection +(-30592, 'MBEDTLS_ERR_SSL_FATAL_ALERT_MESSAGE') +server done +--- instance1 --- +write: b'client data' +drain + +The certificate Common Name (CN) does not match with the expected CN + +client done diff --git a/tests/net_inet/asyncio_tls_open_connection_readline.py b/tests/net_inet/asyncio_tls_open_connection_readline.py new file mode 100644 index 0000000000..70145d91a7 --- /dev/null +++ b/tests/net_inet/asyncio_tls_open_connection_readline.py @@ -0,0 +1,59 @@ +import ssl +import os +import asyncio + +# This certificate was obtained from micropython.org using openssl: +# $ openssl s_client -showcerts -connect micropython.org:443 /dev/null +# The certificate is from Let's Encrypt: +# 1 s:/C=US/O=Let's Encrypt/CN=R3 +# i:/C=US/O=Internet Security Research Group/CN=ISRG Root X1 +# Validity +# Not Before: Sep 4 00:00:00 2020 GMT +# Not After : Sep 15 16:00:00 2025 GMT +# Copy PEM content to a file (certmpy.pem) and convert to DER e.g. +# $ openssl x509 -in certmpy.pem -out certmpy.der -outform DER +# Then convert to hex format, eg using binascii.hexlify(data). + +# Note that the instructions above is to obtain an intermediate +# root CA cert that works for MicroPython. However CPython needs the ultimate root CA +# cert from ISRG + +ca_cert_chain = "isrg.der" + +try: + os.stat(ca_cert_chain) +except OSError: + print("SKIP") + raise SystemExit + +with open(ca_cert_chain, "rb") as ca: + cadata = ca.read() + +client_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) +client_ctx.verify_mode = ssl.CERT_REQUIRED +client_ctx.load_verify_locations(cadata=cadata) + + +async def http_get(url, port, sslctx): + reader, writer = await asyncio.open_connection(url, port, ssl=sslctx) + + print("write GET") + writer.write(b"GET / HTTP/1.0\r\n\r\n") + await writer.drain() + + print("read response") + while True: + data = await reader.readline() + # avoid printing datetime which makes the test fail + if b"GMT" not in data: + print("read:", data) + if not data: + break + + print("close") + writer.close() + await writer.wait_closed() + print("done") + + +asyncio.run(http_get("micropython.org", 443, client_ctx)) diff --git a/tests/net_inet/isrg.der b/tests/net_inet/isrg.der new file mode 100644 index 0000000000000000000000000000000000000000..9d2132e7f1e352fabac7eafb231488b5da91ffb2 GIT binary patch literal 1391 zcmXqLV$C*aVh&!w%*4pVB*@StaDKxjhsTjF$q#lXH+3@@@Un4gwRyCC=VfH%W@Rw& zH{>?pWMd9xVH0Kw4K~y?PzQ0igcUsVN>YpRQcDzqQMi96N{2F6x@sQ zOA8D|4TM2TnT2^ggM-`^g7WiA6e0`_)UeSUJ_S;M+V-u>HW)=goaf7yL{%}fvF;1?F_{JHX*^)7mb z_cWAjyQP1@qPLp4KvBB%lYz~z{&jb6C9i%h=6|S9(7WzD_ly5q%k{o&s`h%|Bc#ex z(95j3;9;=J8{wPpB=-w!_Uf_kT$~tqZ%sS8l;RAn=gy-c5l%vESRjulRoaDHHpQelw1#&mWmj<25Ut_nWV1qwMTG%s)L@ zZ#3Rz-J*5P@#PxEvZ-ABH|}5EDDklY(M=kbokat@+bL(=ez`Qo=d9_8$g;*;h-`WLMh;lRc_g>Iv-DFqo zCF5PpD)i^rs|NwXHO`YuHlHea-Y3t;=GdnK4#`;nE(6$dNYTB&bR(NQ2+$oz?wqHJLsjX!HYm3h*_fBZ@a%uek ze*2NA(-ox)>ah}I#svAgPldH?sMd^L9VXJTe#U|j5E;9$T9Os}&1 zjEw(TSb({M&43@o7Y6ZJ4VZzHfhFz=l^iUlGsD^9O_ z?o;@C)1`#9mMgeli7SS+ehlD?e0}ag-X~KPhVT7{&D4o6YKug*3J5*#Pa(8&H7gpwUsuC^Ywq~GKr43@rUtb$j%*V zXSzC!JAHIpY?|)Bn-;WsJ~s2)HigcR z-KW{sqcnToqipMNtEK|qJDkTmPjj*R=DdjQJNf?H>f^h&YWulf^SYpR=4sI>j;y6q zAB!&hzU1vmo%p4{|F6+t(%W~vdiUeP>Iq_(+2h=TYs}f5dM+QCHs|Whty&MJN;P<_ z^RZ+cWl3o${YKsGG(t*4WP8`@Gk9!gJ;MzXRq} z=D1zmBD#56Ufpb-X;wRebnUN2Km5&csO6u^ip8C`)?_`D(Av1dIWhXO{2lAwvQN4% zdQ0z%8|T;t|E@mm82|syq6>)@52x)|6WeWmz4WT_ftiBq<~klMDs9=v