From a0127919bcdad884160a9534590096894a6ecf62 Mon Sep 17 00:00:00 2001 From: stijn Date: Tue, 24 Oct 2023 17:08:43 +0200 Subject: [PATCH 1/9] tests: Make socket tests compatible with CPython on windows. Note these are mostly the 'bare' socket tests, not the ssl/tls ones for instance: most of these don't run on CPython because of incompatible wrap_socket() arguments. The change mostly consists of checking the WSA error codes next to the errno ones and these are written as numeric values because the names (like WSAEAGAIN) are only available in CPython and not in micropython. Signed-off-by: stijn --- tests/extmod/socket_tcp_basic.py | 2 +- tests/extmod/socket_udp_nonblock.py | 2 +- tests/multi_net/asyncio_tcp_client_rst.py | 2 +- tests/multi_net/asyncio_tcp_client_rst.py.exp | 2 +- tests/multi_net/tcp_accept_recv.py | 2 +- tests/net_hosted/accept_nonblock.py | 2 +- tests/net_hosted/accept_timeout.py | 4 +++- tests/net_hosted/connect_nonblock.py | 2 +- tests/net_inet/ssl_errors.py | 2 +- 9 files changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/extmod/socket_tcp_basic.py b/tests/extmod/socket_tcp_basic.py index ebd30f7862..30d7dbd44e 100644 --- a/tests/extmod/socket_tcp_basic.py +++ b/tests/extmod/socket_tcp_basic.py @@ -11,4 +11,4 @@ s = socket.socket() try: s.recv(1) except OSError as er: - print("ENOTCONN:", er.errno == errno.ENOTCONN) + print("ENOTCONN:", er.errno in (errno.ENOTCONN, 10057)) diff --git a/tests/extmod/socket_udp_nonblock.py b/tests/extmod/socket_udp_nonblock.py index 1e74e2917d..dae2be2e52 100644 --- a/tests/extmod/socket_udp_nonblock.py +++ b/tests/extmod/socket_udp_nonblock.py @@ -18,4 +18,4 @@ s.settimeout(0) try: s.recv(1) except OSError as er: - print("EAGAIN:", er.errno == errno.EAGAIN) + print("EAGAIN:", er.errno in (errno.EAGAIN, 10035)) diff --git a/tests/multi_net/asyncio_tcp_client_rst.py b/tests/multi_net/asyncio_tcp_client_rst.py index af70c2e22e..f2d18ed0b0 100644 --- a/tests/multi_net/asyncio_tcp_client_rst.py +++ b/tests/multi_net/asyncio_tcp_client_rst.py @@ -21,7 +21,7 @@ async def handle_connection(reader, writer): writer.close() await writer.wait_closed() except OSError as er: - print("OSError", er.errno) + print("OSError", er.errno in (22, 104)) ev.set() diff --git a/tests/multi_net/asyncio_tcp_client_rst.py.exp b/tests/multi_net/asyncio_tcp_client_rst.py.exp index 920d1bb8d7..8dee3c14a3 100644 --- a/tests/multi_net/asyncio_tcp_client_rst.py.exp +++ b/tests/multi_net/asyncio_tcp_client_rst.py.exp @@ -1,5 +1,5 @@ --- instance0 --- b'GET / HTTP' -OSError 104 +OSError True --- instance1 --- diff --git a/tests/multi_net/tcp_accept_recv.py b/tests/multi_net/tcp_accept_recv.py index dee14e3b97..d9b700fb62 100644 --- a/tests/multi_net/tcp_accept_recv.py +++ b/tests/multi_net/tcp_accept_recv.py @@ -17,7 +17,7 @@ def instance0(): try: print("recv", s.recv(10)) # should raise Errno 107 ENOTCONN except OSError as er: - print(er.errno in (107, 128)) + print(er.errno in (107, 128, 10057)) s.close() diff --git a/tests/net_hosted/accept_nonblock.py b/tests/net_hosted/accept_nonblock.py index 30d2033e65..915174a808 100644 --- a/tests/net_hosted/accept_nonblock.py +++ b/tests/net_hosted/accept_nonblock.py @@ -9,5 +9,5 @@ s.listen(1) try: s.accept() except OSError as er: - print(er.errno == 11) # 11 is EAGAIN + print(er.errno in (11, 10035)) # 11 is EAGAIN s.close() diff --git a/tests/net_hosted/accept_timeout.py b/tests/net_hosted/accept_timeout.py index 865d2aad26..2f047ee843 100644 --- a/tests/net_hosted/accept_timeout.py +++ b/tests/net_hosted/accept_timeout.py @@ -15,5 +15,7 @@ s.listen(1) try: s.accept() except OSError as er: - print(er.errno in (errno.ETIMEDOUT, "timed out")) # CPython uses a string instead of errno + print( + er.errno == errno.ETIMEDOUT or str(er) == "timed out" + ) # CPython uses a string instead of errno s.close() diff --git a/tests/net_hosted/connect_nonblock.py b/tests/net_hosted/connect_nonblock.py index 781f1a4ee2..4640dd5808 100644 --- a/tests/net_hosted/connect_nonblock.py +++ b/tests/net_hosted/connect_nonblock.py @@ -9,7 +9,7 @@ def test(peer_addr): try: s.connect(peer_addr) except OSError as er: - print(er.errno == errno.EINPROGRESS) + print(er.errno in (errno.EINPROGRESS, 10035)) s.close() diff --git a/tests/net_inet/ssl_errors.py b/tests/net_inet/ssl_errors.py index bc4e5910bc..5798a62f46 100644 --- a/tests/net_inet/ssl_errors.py +++ b/tests/net_inet/ssl_errors.py @@ -12,7 +12,7 @@ def test(addr, hostname, block=True): s.connect(addr) print("connected") except OSError as e: - if e.errno != errno.EINPROGRESS: + if e.errno not in (errno.EINPROGRESS, 10035): raise print("EINPROGRESS") From 07f229e03063e254f16e3d4ccf33ebe90d789502 Mon Sep 17 00:00:00 2001 From: stijn Date: Wed, 25 Oct 2023 11:55:38 +0200 Subject: [PATCH 2/9] tests: Add socket timeout tests for connect and recv. Signed-off-by: stijn --- tests/multi_net/tcp_recv_timeout.py | 37 +++++++++++++++++++++++++ tests/multi_net/tcp_recv_timeout.py.exp | 6 ++++ tests/net_hosted/connect_timeout.py | 16 +++++++++++ tests/net_hosted/connect_timeout.py.exp | 1 + 4 files changed, 60 insertions(+) create mode 100644 tests/multi_net/tcp_recv_timeout.py create mode 100644 tests/multi_net/tcp_recv_timeout.py.exp create mode 100644 tests/net_hosted/connect_timeout.py create mode 100644 tests/net_hosted/connect_timeout.py.exp diff --git a/tests/multi_net/tcp_recv_timeout.py b/tests/multi_net/tcp_recv_timeout.py new file mode 100644 index 0000000000..415ee9dff1 --- /dev/null +++ b/tests/multi_net/tcp_recv_timeout.py @@ -0,0 +1,37 @@ +import errno +import socket + +PORT = 8000 + + +# Server +def instance0(): + multitest.globals(IP=multitest.get_network_ip()) + s = socket.socket() + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(socket.getaddrinfo("0.0.0.0", PORT)[0][-1]) + s.listen() + multitest.next() + s2, _ = s.accept() + s2.settimeout(0.2) + try: + s2.recv(1) + except OSError as er: + print(er.errno in (errno.ETIMEDOUT, errno.EAGAIN) or str(er) == "timed out") + multitest.next() + s2.close() + s.close() + + +# Client +def instance1(): + multitest.next() + s = socket.socket() + s.connect(socket.getaddrinfo(IP, PORT)[0][-1]) + s.settimeout(0.2) + try: + s.recv(1) + except OSError as er: + print(er.errno in (errno.ETIMEDOUT, errno.EAGAIN) or str(er) == "timed out") + multitest.next() + s.close() diff --git a/tests/multi_net/tcp_recv_timeout.py.exp b/tests/multi_net/tcp_recv_timeout.py.exp new file mode 100644 index 0000000000..6c19840fdd --- /dev/null +++ b/tests/multi_net/tcp_recv_timeout.py.exp @@ -0,0 +1,6 @@ +--- instance0 --- +True +NEXT +--- instance1 --- +True +NEXT diff --git a/tests/net_hosted/connect_timeout.py b/tests/net_hosted/connect_timeout.py new file mode 100644 index 0000000000..1dc259d306 --- /dev/null +++ b/tests/net_hosted/connect_timeout.py @@ -0,0 +1,16 @@ +import errno +import socket + + +def test(peer_addr): + s = socket.socket() + s.settimeout(1) + try: + s.connect(peer_addr) + except OSError as er: + print(er.errno == errno.ETIMEDOUT or str(er) == "timed out") + s.close() + + +if __name__ == "__main__": + test(socket.getaddrinfo("192.0.0.0", 8888)[0][-1]) diff --git a/tests/net_hosted/connect_timeout.py.exp b/tests/net_hosted/connect_timeout.py.exp new file mode 100644 index 0000000000..0ca95142bb --- /dev/null +++ b/tests/net_hosted/connect_timeout.py.exp @@ -0,0 +1 @@ +True From ad908fa9bd24b25d7becc4911c7ec134baadd44a Mon Sep 17 00:00:00 2001 From: stijn Date: Wed, 25 Oct 2023 16:53:28 +0200 Subject: [PATCH 3/9] unix/modsocket: Introduce typedefs and macros. Upcoming commits are going to port this file to work with sockets on windows so prepare for that with the platform-specific bits. Signed-off-by: stijn --- ports/unix/modsocket.c | 68 ++++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/ports/unix/modsocket.c b/ports/unix/modsocket.c index 6d6059ae44..22eb6496f9 100644 --- a/ports/unix/modsocket.c +++ b/ports/unix/modsocket.c @@ -53,6 +53,16 @@ #include "extmod/vfs.h" #include +typedef socklen_t sock_len_t; +typedef int socket_t; +typedef ssize_t socket_size_t; + +#define socket_errno errno +#define socket_eintr EINTR +#define read_socket read +#define write_socket write +#define close_socket close + /* The idea of this module is to implement reasonable minimum of socket-related functions to write typical clients and servers. @@ -71,18 +81,18 @@ // fields should have the same layout. typedef struct _mp_obj_socket_t { mp_obj_base_t base; - int fd; + socket_t fd; bool blocking; } mp_obj_socket_t; const mp_obj_type_t mp_type_socket; // Helper functions -static inline mp_obj_t mp_obj_from_sockaddr(const struct sockaddr *addr, socklen_t len) { +static inline mp_obj_t mp_obj_from_sockaddr(const struct sockaddr *addr, sock_len_t len) { return mp_obj_new_bytes((const byte *)addr, len); } -static mp_obj_socket_t *socket_new(int fd) { +static mp_obj_socket_t *socket_new(socket_t fd) { mp_obj_socket_t *o = mp_obj_malloc(mp_obj_socket_t, &mp_type_socket); o->fd = fd; o->blocking = true; @@ -98,8 +108,8 @@ static void socket_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kin static mp_uint_t socket_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errcode) { mp_obj_socket_t *o = MP_OBJ_TO_PTR(o_in); - ssize_t r; - MP_HAL_RETRY_SYSCALL(r, read(o->fd, buf, size), { + socket_size_t r; + MP_HAL_RETRY_SYSCALL(r, read_socket(o->fd, buf, size), { // On blocking socket, we get EAGAIN in case SO_RCVTIMEO/SO_SNDTIMEO // timed out, and need to convert that to ETIMEDOUT. if (err == EAGAIN && o->blocking) { @@ -114,8 +124,8 @@ static mp_uint_t socket_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errc static mp_uint_t socket_write(mp_obj_t o_in, const void *buf, mp_uint_t size, int *errcode) { mp_obj_socket_t *o = MP_OBJ_TO_PTR(o_in); - ssize_t r; - MP_HAL_RETRY_SYSCALL(r, write(o->fd, buf, size), { + socket_size_t r; + MP_HAL_RETRY_SYSCALL(r, write_socket(o->fd, buf, size), { // On blocking socket, we get EAGAIN in case SO_RCVTIMEO/SO_SNDTIMEO // timed out, and need to convert that to ETIMEDOUT. if (err == EAGAIN && o->blocking) { @@ -141,7 +151,7 @@ static mp_uint_t socket_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, i // file descriptor. If you're interested to catch I/O errors before // closing fd, fsync() it. MP_THREAD_GIL_EXIT(); - close(self->fd); + close_socket(self->fd); MP_THREAD_GIL_ENTER(); return 0; @@ -204,9 +214,9 @@ static mp_obj_t socket_connect(mp_obj_t self_in, mp_obj_t addr_in) { int r = connect(self->fd, (const struct sockaddr *)bufinfo.buf, bufinfo.len); MP_THREAD_GIL_ENTER(); if (r == -1) { - int err = errno; + int err = socket_errno; if (self->blocking) { - if (err == EINTR) { + if (err == socket_eintr) { mp_handle_pending(true); continue; } @@ -229,7 +239,7 @@ static mp_obj_t socket_bind(mp_obj_t self_in, mp_obj_t addr_in) { MP_THREAD_GIL_EXIT(); int r = bind(self->fd, (const struct sockaddr *)bufinfo.buf, bufinfo.len); MP_THREAD_GIL_ENTER(); - RAISE_ERRNO(r, errno); + RAISE_ERRNO(r, socket_errno); return mp_const_none; } static MP_DEFINE_CONST_FUN_OBJ_2(socket_bind_obj, socket_bind); @@ -247,7 +257,7 @@ static mp_obj_t socket_listen(size_t n_args, const mp_obj_t *args) { MP_THREAD_GIL_EXIT(); int r = listen(self->fd, backlog); MP_THREAD_GIL_ENTER(); - RAISE_ERRNO(r, errno); + RAISE_ERRNO(r, socket_errno); return mp_const_none; } static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(socket_listen_obj, 1, 2, socket_listen); @@ -257,8 +267,8 @@ static mp_obj_t socket_accept(mp_obj_t self_in) { // sockaddr_storage isn't stack-friendly (129 bytes or so) // struct sockaddr_storage addr; byte addr[32]; - socklen_t addr_len = sizeof(addr); - int fd; + sock_len_t addr_len = sizeof(addr); + socket_t fd; MP_HAL_RETRY_SYSCALL(fd, accept(self->fd, (struct sockaddr *)&addr, &addr_len), { // EAGAIN on a blocking socket means the operation timed out if (self->blocking && err == EAGAIN) { @@ -288,7 +298,7 @@ static mp_obj_t socket_recv(size_t n_args, const mp_obj_t *args) { } byte *buf = m_new(byte, sz); - ssize_t out_sz; + socket_size_t out_sz; MP_HAL_RETRY_SYSCALL(out_sz, recv(self->fd, buf, sz, flags), mp_raise_OSError(err)); mp_obj_t ret = mp_obj_new_str_of_type(&mp_type_bytes, buf, out_sz); m_del(char, buf, sz); @@ -306,10 +316,10 @@ static mp_obj_t socket_recvfrom(size_t n_args, const mp_obj_t *args) { } struct sockaddr_storage addr; - socklen_t addr_len = sizeof(addr); + sock_len_t addr_len = sizeof(addr); byte *buf = m_new(byte, sz); - ssize_t out_sz; + socket_size_t out_sz; MP_HAL_RETRY_SYSCALL(out_sz, recvfrom(self->fd, buf, sz, flags, (struct sockaddr *)&addr, &addr_len), mp_raise_OSError(err)); mp_obj_t buf_o = mp_obj_new_str_of_type(&mp_type_bytes, buf, out_sz); @@ -336,7 +346,7 @@ static mp_obj_t socket_send(size_t n_args, const mp_obj_t *args) { mp_buffer_info_t bufinfo; mp_get_buffer_raise(args[1], &bufinfo, MP_BUFFER_READ); - ssize_t out_sz; + socket_size_t out_sz; MP_HAL_RETRY_SYSCALL(out_sz, send(self->fd, bufinfo.buf, bufinfo.len, flags), mp_raise_OSError(err)); return MP_OBJ_NEW_SMALL_INT(out_sz); @@ -356,7 +366,7 @@ static mp_obj_t socket_sendto(size_t n_args, const mp_obj_t *args) { mp_buffer_info_t bufinfo, addr_bi; mp_get_buffer_raise(args[1], &bufinfo, MP_BUFFER_READ); mp_get_buffer_raise(dst_addr, &addr_bi, MP_BUFFER_READ); - ssize_t out_sz; + socket_size_t out_sz; MP_HAL_RETRY_SYSCALL(out_sz, sendto(self->fd, bufinfo.buf, bufinfo.len, flags, (struct sockaddr *)addr_bi.buf, addr_bi.len), mp_raise_OSError(err)); return MP_OBJ_NEW_SMALL_INT(out_sz); @@ -370,7 +380,7 @@ static mp_obj_t socket_setsockopt(size_t n_args, const mp_obj_t *args) { int option = mp_obj_get_int(args[2]); const void *optval; - socklen_t optlen; + sock_len_t optlen; int val; if (mp_obj_is_int(args[3])) { val = mp_obj_int_get_truncated(args[3]); @@ -385,7 +395,7 @@ static mp_obj_t socket_setsockopt(size_t n_args, const mp_obj_t *args) { MP_THREAD_GIL_EXIT(); int r = setsockopt(self->fd, level, option, optval, optlen); MP_THREAD_GIL_ENTER(); - RAISE_ERRNO(r, errno); + RAISE_ERRNO(r, socket_errno); return mp_const_none; } static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(socket_setsockopt_obj, 4, 4, socket_setsockopt); @@ -397,7 +407,7 @@ static mp_obj_t socket_setblocking(mp_obj_t self_in, mp_obj_t flag_in) { int flags = fcntl(self->fd, F_GETFL, 0); if (flags == -1) { MP_THREAD_GIL_ENTER(); - RAISE_ERRNO(flags, errno); + RAISE_ERRNO(flags, socket_errno); } if (val) { flags &= ~O_NONBLOCK; @@ -406,7 +416,7 @@ static mp_obj_t socket_setblocking(mp_obj_t self_in, mp_obj_t flag_in) { } flags = fcntl(self->fd, F_SETFL, flags); MP_THREAD_GIL_ENTER(); - RAISE_ERRNO(flags, errno); + RAISE_ERRNO(flags, socket_errno); self->blocking = val; return mp_const_none; } @@ -442,11 +452,11 @@ static mp_obj_t socket_settimeout(mp_obj_t self_in, mp_obj_t timeout_in) { r = setsockopt(self->fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(struct timeval)); if (r == -1) { MP_THREAD_GIL_ENTER(); - RAISE_ERRNO(r, errno); + RAISE_ERRNO(r, socket_errno); } r = setsockopt(self->fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(struct timeval)); MP_THREAD_GIL_ENTER(); - RAISE_ERRNO(r, errno); + RAISE_ERRNO(r, socket_errno); } if (self->blocking != new_blocking) { @@ -491,9 +501,9 @@ static mp_obj_t socket_make_new(const mp_obj_type_t *type_in, size_t n_args, siz } MP_THREAD_GIL_EXIT(); - int fd = socket(family, type, proto); + socket_t fd = socket(family, type, proto); MP_THREAD_GIL_ENTER(); - RAISE_ERRNO(fd, errno); + RAISE_ERRNO(fd, socket_errno); return MP_OBJ_FROM_PTR(socket_new(fd)); } @@ -541,7 +551,7 @@ static mp_obj_t mod_socket_inet_pton(mp_obj_t family_in, mp_obj_t addr_in) { int family = mp_obj_get_int(family_in); byte binaddr[BINADDR_MAX_LEN]; int r = inet_pton(family, mp_obj_str_get_str(addr_in), binaddr); - RAISE_ERRNO(r, errno); + RAISE_ERRNO(r, socket_errno); if (r == 0) { mp_raise_OSError(MP_EINVAL); } @@ -565,7 +575,7 @@ static mp_obj_t mod_socket_inet_ntop(mp_obj_t family_in, mp_obj_t binaddr_in) { vstr_t vstr; vstr_init_len(&vstr, family == AF_INET ? INET_ADDRSTRLEN : INET6_ADDRSTRLEN); if (inet_ntop(family, bufinfo.buf, vstr.buf, vstr.len) == NULL) { - mp_raise_OSError(errno); + mp_raise_OSError(socket_errno); } vstr.len = strlen(vstr.buf); return mp_obj_new_str_from_utf8_vstr(&vstr); From f75131099beda503102bfee12b5c8876ded0bf15 Mon Sep 17 00:00:00 2001 From: stijn Date: Tue, 24 Oct 2023 15:26:00 +0200 Subject: [PATCH 4/9] unix/modsocket: Remove obsolete comment. Signed-off-by: stijn --- ports/unix/modsocket.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/ports/unix/modsocket.c b/ports/unix/modsocket.c index 22eb6496f9..c4306c049e 100644 --- a/ports/unix/modsocket.c +++ b/ports/unix/modsocket.c @@ -77,8 +77,6 @@ typedef ssize_t socket_size_t; should be add to separate modules (C or Python level). */ -// This type must "inherit" from mp_obj_fdfile_t, i.e. matching subset of -// fields should have the same layout. typedef struct _mp_obj_socket_t { mp_obj_base_t base; socket_t fd; From 170c04dcaa528d5c5dd9663dc5c73cf630c716bd Mon Sep 17 00:00:00 2001 From: stijn Date: Wed, 25 Oct 2023 13:42:52 +0200 Subject: [PATCH 5/9] windows: Implement socket module. Signed-off-by: stijn --- ports/unix/modsocket.c | 182 +++++++++++++++++++++++++++++- ports/windows/Makefile | 1 + ports/windows/init.c | 12 ++ ports/windows/micropython.vcxproj | 1 + ports/windows/mpconfigport.h | 1 + tests/run-tests.py | 5 + 6 files changed, 196 insertions(+), 6 deletions(-) diff --git a/ports/unix/modsocket.c b/ports/unix/modsocket.c index c4306c049e..cb067dc622 100644 --- a/ports/unix/modsocket.c +++ b/ports/unix/modsocket.c @@ -36,10 +36,26 @@ #include #include #include +#ifdef _WIN32 +// To get inet_pton and inet_ntop. +#ifdef __MINGW32__ +#ifdef _WIN32_WINNT +#undef _WIN32_WINNT +#endif +#define _WIN32_WINNT 0x0600 +#endif +#include +#include +#ifndef __MINGW32__ +#pragma comment(lib, "Ws2_32.lib") +#endif +#else #include #include #include #include +#include +#endif #include #include @@ -51,8 +67,97 @@ #include "py/mphal.h" #include "py/mpthread.h" #include "extmod/vfs.h" -#include +#ifdef _WIN32 +// Some extra info regarding the windows-specific code: +// - like CPython, most of the error codes raised as OSError will be the WSA error codes, not errno. +// - places where EAGAIN is turned into MP_ETIMEOUT are generally not needed On windows since the +// socket calls already return WSAETIMEDOUT. +// - makefile() and poll functionality are not implemented. + +typedef int sock_len_t; +typedef int socket_size_t; +typedef SOCKET socket_t; + +// Just to make sure since we rely on this. +#if SOCKET_ERROR != -1 +#error socket functions must return -1 for errors +#endif +#if NO_ERROR != 0 +#error socket functions must return 0 for no errors +#endif + +#define socket_errno WSAGetLastError() +#define socket_eintr WSAEINTR +#define read_socket(fd, buf, size) recv(fd, buf, size, 0) +#define write_socket(fd, buf, size) send(fd, buf, size, 0) +#define close_socket(fd) closesocket(fd) + +void wsa_startup() { + WSADATA wsaData; + (void)WSAStartup(MAKEWORD(1, 1), &wsaData); +} + +void wsa_cleanup() { + (void)WSACleanup(); +} + +// Socket calls are syscalls but with WSA error codes, not errno. +#ifdef MP_HAL_RETRY_SYSCALL +#undef MP_HAL_RETRY_SYSCALL +#endif + +#define MP_HAL_RETRY_SYSCALL(ret, syscall, raise) { \ + for (;;) { \ + MP_THREAD_GIL_EXIT(); \ + ret = syscall; \ + MP_THREAD_GIL_ENTER(); \ + if (ret == -1) { \ + int err = WSAGetLastError(); \ + if (err == WSAEINTR) { \ + mp_handle_pending(true); \ + continue; \ + } \ + raise; \ + } \ + break; \ + } \ +} + +// Get SO_RCVTIMEO or SO_SNDTIMEO values. +DWORD get_socket_timeout(socket_t sock, bool read_or_write) { + MP_THREAD_GIL_EXIT(); + DWORD timeout = 0; + int opt_len = sizeof(timeout); + const int opt_name = read_or_write ? SO_RCVTIMEO : SO_SNDTIMEO; + const int r = getsockopt(sock, SOL_SOCKET, opt_name, (char *)&timeout, &opt_len); + MP_THREAD_GIL_ENTER(); + RAISE_ERRNO(r, WSAGetLastError()); + return timeout; +} + +// Perform select() call, raising timeout error if timed out. +void select_on_socket(socket_t sock, bool read_or_write, DWORD timeout) { + fd_set fdset; + FD_ZERO(&fdset); + FD_SET(sock, &fdset); + struct timeval tv; + tv.tv_sec = timeout / 1000; + tv.tv_usec = (timeout * 1000) % 1000; + int r; + fd_set *read_fd = read_or_write ? &fdset : NULL; + fd_set *write_fd = read_or_write ? NULL : &fdset; + // First argument is ignored: "The nfds parameter is included only for + // compatibility with Berkeley sockets". + MP_HAL_RETRY_SYSCALL(r, select(1, read_fd, write_fd, NULL, &tv), { + // r < 0 is error, r == 0 is timeout, r > 0 = no timeout. + RAISE_ERRNO(r, WSAGetLastError()); + }); + if (r == 0) { + mp_raise_OSError(MP_ETIMEDOUT); + } +} +#else typedef socklen_t sock_len_t; typedef int socket_t; typedef ssize_t socket_size_t; @@ -63,6 +168,9 @@ typedef ssize_t socket_size_t; #define write_socket write #define close_socket close +#define initialize_socket_system() {} +#endif + /* The idea of this module is to implement reasonable minimum of socket-related functions to write typical clients and servers. @@ -153,6 +261,7 @@ static mp_uint_t socket_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, i MP_THREAD_GIL_ENTER(); return 0; + #ifndef _WIN32 case MP_STREAM_GET_FILENO: return self->fd; @@ -187,6 +296,7 @@ static mp_uint_t socket_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, i return ret; } #endif + #endif default: *errcode = MP_EINVAL; @@ -194,11 +304,13 @@ static mp_uint_t socket_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, i } } +#ifndef _WIN32 static mp_obj_t socket_fileno(mp_obj_t self_in) { mp_obj_socket_t *self = MP_OBJ_TO_PTR(self_in); return MP_OBJ_NEW_SMALL_INT(self->fd); } static MP_DEFINE_CONST_FUN_OBJ_1(socket_fileno_obj, socket_fileno); +#endif static mp_obj_t socket_connect(mp_obj_t self_in, mp_obj_t addr_in) { mp_obj_socket_t *self = MP_OBJ_TO_PTR(self_in); @@ -208,6 +320,15 @@ static mp_obj_t socket_connect(mp_obj_t self_in, mp_obj_t addr_in) { // special case of PEP 475 to retry only if blocking so we can't use // MP_HAL_RETRY_SYSCALL() here for (;;) { + #ifdef _WIN32 + // The connect() call has no timeout so implement it by first calling select(). + // In theory there's a race condition here between select() returning and connect() + // being called, in practice doesn't seem worth fixing. + const DWORD timeout = get_socket_timeout(self->fd, true); + if (timeout > 0 && self->blocking) { + select_on_socket(self->fd, true, timeout); + } + #endif MP_THREAD_GIL_EXIT(); int r = connect(self->fd, (const struct sockaddr *)bufinfo.buf, bufinfo.len); MP_THREAD_GIL_ENTER(); @@ -267,6 +388,26 @@ static mp_obj_t socket_accept(mp_obj_t self_in) { byte addr[32]; sock_len_t addr_len = sizeof(addr); socket_t fd; + #ifdef _WIN32 + // The accept() call has no timeout so manually implement it by first calling select(). + const DWORD timeout = get_socket_timeout(self->fd, true); + if (timeout > 0 && self->blocking) { + select_on_socket(self->fd, true, timeout); + } + // The accept() call returns a socket, not an int, so cannot use MP_HAL_RETRY_SYSCALL. + for (;;) { + fd = accept(self->fd, (struct sockaddr *)&addr, &addr_len); + if (fd == INVALID_SOCKET) { + const int err = WSAGetLastError(); + if (err == WSAEINTR) { + mp_handle_pending(1); + continue; + } + mp_raise_OSError(err); + } + break; + } + #else MP_HAL_RETRY_SYSCALL(fd, accept(self->fd, (struct sockaddr *)&addr, &addr_len), { // EAGAIN on a blocking socket means the operation timed out if (self->blocking && err == EAGAIN) { @@ -274,6 +415,7 @@ static mp_obj_t socket_accept(mp_obj_t self_in) { } mp_raise_OSError(err); }); + #endif mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(2, NULL)); t->items[0] = MP_OBJ_FROM_PTR(socket_new(fd)); @@ -297,7 +439,7 @@ static mp_obj_t socket_recv(size_t n_args, const mp_obj_t *args) { byte *buf = m_new(byte, sz); socket_size_t out_sz; - MP_HAL_RETRY_SYSCALL(out_sz, recv(self->fd, buf, sz, flags), mp_raise_OSError(err)); + MP_HAL_RETRY_SYSCALL(out_sz, recv(self->fd, (char *)buf, sz, flags), mp_raise_OSError(err)); mp_obj_t ret = mp_obj_new_str_of_type(&mp_type_bytes, buf, out_sz); m_del(char, buf, sz); return ret; @@ -318,7 +460,7 @@ static mp_obj_t socket_recvfrom(size_t n_args, const mp_obj_t *args) { byte *buf = m_new(byte, sz); socket_size_t out_sz; - MP_HAL_RETRY_SYSCALL(out_sz, recvfrom(self->fd, buf, sz, flags, (struct sockaddr *)&addr, &addr_len), + MP_HAL_RETRY_SYSCALL(out_sz, recvfrom(self->fd, (char *)buf, sz, flags, (struct sockaddr *)&addr, &addr_len), mp_raise_OSError(err)); mp_obj_t buf_o = mp_obj_new_str_of_type(&mp_type_bytes, buf, out_sz); m_del(char, buf, sz); @@ -402,6 +544,11 @@ static mp_obj_t socket_setblocking(mp_obj_t self_in, mp_obj_t flag_in) { mp_obj_socket_t *self = MP_OBJ_TO_PTR(self_in); int val = mp_obj_is_true(flag_in); MP_THREAD_GIL_EXIT(); + #ifdef _WIN32 + u_long mode = val ? 0 : 1; + // Called 'flags' but here it's just the return value. + const int flags = ioctlsocket(self->fd, FIONBIO, &mode); + #else int flags = fcntl(self->fd, F_GETFL, 0); if (flags == -1) { MP_THREAD_GIL_ENTER(); @@ -413,6 +560,7 @@ static mp_obj_t socket_setblocking(mp_obj_t self_in, mp_obj_t flag_in) { flags |= O_NONBLOCK; } flags = fcntl(self->fd, F_SETFL, flags); + #endif MP_THREAD_GIL_ENTER(); RAISE_ERRNO(flags, socket_errno); self->blocking = val; @@ -423,6 +571,14 @@ static MP_DEFINE_CONST_FUN_OBJ_2(socket_setblocking_obj, socket_setblocking); static mp_obj_t socket_settimeout(mp_obj_t self_in, mp_obj_t timeout_in) { mp_obj_socket_t *self = MP_OBJ_TO_PTR(self_in); struct timeval tv = {0, }; + #ifdef _WIN32 + DWORD time_val = 0; + const char *ptv = (const char *)&time_val; + const int opt_len = sizeof(DWORD); + #else + struct timeval *ptv = &tv; + const int opt_len = sizeof(struct timeval); + #endif bool new_blocking = true; // Timeout of None means no timeout, which in POSIX is signified with 0 timeout, @@ -436,6 +592,9 @@ static mp_obj_t socket_settimeout(mp_obj_t self_in, mp_obj_t timeout_in) { #else tv.tv_sec = mp_obj_get_int(timeout_in); #endif + #ifdef _WIN32 + time_val = tv.tv_usec / 1000 + tv.tv_sec * 1000; + #endif // For SO_RCVTIMEO/SO_SNDTIMEO, zero timeout means infinity, but // for Python API it means non-blocking. @@ -447,12 +606,12 @@ static mp_obj_t socket_settimeout(mp_obj_t self_in, mp_obj_t timeout_in) { if (new_blocking) { int r; MP_THREAD_GIL_EXIT(); - r = setsockopt(self->fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(struct timeval)); + r = setsockopt(self->fd, SOL_SOCKET, SO_RCVTIMEO, ptv, opt_len); if (r == -1) { MP_THREAD_GIL_ENTER(); RAISE_ERRNO(r, socket_errno); } - r = setsockopt(self->fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(struct timeval)); + r = setsockopt(self->fd, SOL_SOCKET, SO_SNDTIMEO, ptv, opt_len); MP_THREAD_GIL_ENTER(); RAISE_ERRNO(r, socket_errno); } @@ -465,6 +624,7 @@ static mp_obj_t socket_settimeout(mp_obj_t self_in, mp_obj_t timeout_in) { } static MP_DEFINE_CONST_FUN_OBJ_2(socket_settimeout_obj, socket_settimeout); +#ifndef _WIN32 static mp_obj_t socket_makefile(size_t n_args, const mp_obj_t *args) { // TODO: CPython explicitly says that closing returned object doesn't close // the original socket (Python2 at all says that fd is dup()ed). But we @@ -476,6 +636,7 @@ static mp_obj_t socket_makefile(size_t n_args, const mp_obj_t *args) { return mp_vfs_open(n_args, new_args, (mp_map_t *)&mp_const_empty_map); } static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(socket_makefile_obj, 1, 3, socket_makefile); +#endif static mp_obj_t socket_make_new(const mp_obj_type_t *type_in, size_t n_args, size_t n_kw, const mp_obj_t *args) { (void)type_in; @@ -500,14 +661,21 @@ static mp_obj_t socket_make_new(const mp_obj_type_t *type_in, size_t n_args, siz MP_THREAD_GIL_EXIT(); socket_t fd = socket(family, type, proto); + #ifdef _WIN32 + const int err = fd == INVALID_SOCKET ? -1 : 0; + #else + const int err = fd; + #endif MP_THREAD_GIL_ENTER(); - RAISE_ERRNO(fd, socket_errno); + RAISE_ERRNO(err, socket_errno); return MP_OBJ_FROM_PTR(socket_new(fd)); } static const mp_rom_map_elem_t socket_locals_dict_table[] = { + #ifndef _WIN32 { MP_ROM_QSTR(MP_QSTR_fileno), MP_ROM_PTR(&socket_fileno_obj) }, { MP_ROM_QSTR(MP_QSTR_makefile), MP_ROM_PTR(&socket_makefile_obj) }, + #endif { MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) }, { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) }, { MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) }, @@ -708,7 +876,9 @@ static const mp_rom_map_elem_t mp_module_socket_globals_table[] = { C(SOCK_RAW), C(MSG_DONTROUTE), + #ifndef _WIN32 C(MSG_DONTWAIT), + #endif C(SOL_SOCKET), C(SO_BROADCAST), diff --git a/ports/windows/Makefile b/ports/windows/Makefile index bb635167da..4d0622e8a4 100644 --- a/ports/windows/Makefile +++ b/ports/windows/Makefile @@ -56,6 +56,7 @@ SRC_C = \ shared/runtime/gchelper_generic.c \ ports/unix/main.c \ ports/unix/input.c \ + ports/unix/modsocket.c \ ports/unix/gccollect.c \ windows_mphal.c \ realpath.c \ diff --git a/ports/windows/init.c b/ports/windows/init.c index 87d581c304..fba8d2c429 100644 --- a/ports/windows/init.c +++ b/ports/windows/init.c @@ -24,6 +24,7 @@ * THE SOFTWARE. */ +#include "py/mpconfig.h" #include #include #include @@ -34,6 +35,11 @@ extern BOOL WINAPI console_sighandler(DWORD evt); +#if MICROPY_PY_SOCKET +extern void wsa_startup(); +extern void wsa_cleanup(); +#endif + #ifdef _MSC_VER void invalid_param_handler(const wchar_t *expr, const wchar_t *fun, const wchar_t *file, unsigned int line, uintptr_t p) { } @@ -61,8 +67,14 @@ void init() { _set_output_format(_TWO_DIGIT_EXPONENT); #endif set_fmode_binary(); + #if MICROPY_PY_SOCKET + wsa_startup(); + #endif } void deinit() { SetConsoleCtrlHandler(console_sighandler, FALSE); + #if MICROPY_PY_SOCKET + wsa_cleanup(); + #endif } diff --git a/ports/windows/micropython.vcxproj b/ports/windows/micropython.vcxproj index 9326f3f4cd..bd647f6736 100644 --- a/ports/windows/micropython.vcxproj +++ b/ports/windows/micropython.vcxproj @@ -94,6 +94,7 @@ + diff --git a/ports/windows/mpconfigport.h b/ports/windows/mpconfigport.h index 55e44c6f5c..05bfb4e145 100644 --- a/ports/windows/mpconfigport.h +++ b/ports/windows/mpconfigport.h @@ -107,6 +107,7 @@ #define MICROPY_PY_ARRAY_SLICE_ASSIGN (1) #define MICROPY_PY_BUILTINS_SLICE_ATTRS (1) #define MICROPY_PY_SYS_PATH_ARGV_DEFAULTS (0) +#define MICROPY_PY_SOCKET (1) #define MICROPY_PY_SYS_EXIT (1) #define MICROPY_PY_SYS_ATEXIT (1) #define MICROPY_PY_SYS_PLATFORM "win32" diff --git a/tests/run-tests.py b/tests/run-tests.py index 4f55cdd398..a608afc103 100755 --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -700,6 +700,11 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): # Some tests use unsupported features on Windows if os.name == "nt": skip_tests.add("import/import_file.py") # works but CPython prints forward slashes + # Socket + select is not implemented. + skip_tests.add("extmod/select_ipoll.py") + skip_tests.add("extmod/select_poll_basic.py") + skip_tests.add("extmod/select_poll_custom.py") + skip_tests.add("extmod/select_poll_udp.py") # Some tests are known to fail with native emitter # Remove them from the below when they work From c1301b43dba4c9d516df4e5b4a64dc8c6d94cb88 Mon Sep 17 00:00:00 2001 From: stijn Date: Mon, 25 Mar 2024 14:33:50 +0100 Subject: [PATCH 6/9] windows/msvc: Add a build target for updating submodules. This is more consistent with the Makefile/CMake-based builds. Signed-off-by: stijn --- .github/workflows/ports_windows.yml | 2 +- ports/windows/README.md | 1 + ports/windows/micropython.vcxproj | 2 +- ports/windows/msvc/common.props | 12 ++++++++++++ ports/windows/variants/dev/mpconfigvariant.props | 3 +++ .../windows/variants/standard/mpconfigvariant.props | 3 +++ 6 files changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ports_windows.yml b/.github/workflows/ports_windows.yml index be2a2a8dac..6d94c9c808 100644 --- a/.github/workflows/ports_windows.yml +++ b/.github/workflows/ports_windows.yml @@ -61,7 +61,7 @@ jobs: - name: Build mpy-cross.exe run: msbuild mpy-cross\mpy-cross.vcxproj -maxcpucount -property:Configuration=${{ matrix.configuration }} -property:Platform=${{ matrix.platform }} - name: Update submodules - run: git submodule update --init lib/micropython-lib + run: msbuild ports\windows\micropython.vcxproj -target:UpdateSubmodules -property:Configuration=${{ matrix.configuration }} -property:Platform=${{ matrix.platform }} -property:PyVariant=${{ matrix.variant }} - name: Build micropython.exe run: msbuild ports\windows\micropython.vcxproj -maxcpucount -property:Configuration=${{ matrix.configuration }} -property:Platform=${{ matrix.platform }} -property:PyVariant=${{ matrix.variant }} - name: Get micropython.exe path diff --git a/ports/windows/README.md b/ports/windows/README.md index 2b3ed44599..268d5f9962 100644 --- a/ports/windows/README.md +++ b/ports/windows/README.md @@ -69,6 +69,7 @@ In the IDE, open `micropython-cross.vcxproj` and `micropython.vcxproj` and build To build from the command line: msbuild ../../mpy-cross/mpy-cross.vcxproj + msbuild micropython.vcxproj /t:UpdateSubmodules msbuild micropython.vcxproj __Variants__ diff --git a/ports/windows/micropython.vcxproj b/ports/windows/micropython.vcxproj index bd647f6736..462821a6a0 100644 --- a/ports/windows/micropython.vcxproj +++ b/ports/windows/micropython.vcxproj @@ -108,7 +108,7 @@ - + diff --git a/ports/windows/msvc/common.props b/ports/windows/msvc/common.props index 55c9c934f2..ecb6641371 100644 --- a/ports/windows/msvc/common.props +++ b/ports/windows/msvc/common.props @@ -62,4 +62,16 @@ + + + + + + + + + + diff --git a/ports/windows/variants/dev/mpconfigvariant.props b/ports/windows/variants/dev/mpconfigvariant.props index be1f1ae51e..be1ee1eb29 100644 --- a/ports/windows/variants/dev/mpconfigvariant.props +++ b/ports/windows/variants/dev/mpconfigvariant.props @@ -8,4 +8,7 @@ %(PreprocessorDefinitions);MICROPY_ROM_TEXT_COMPRESSION=1 + + + diff --git a/ports/windows/variants/standard/mpconfigvariant.props b/ports/windows/variants/standard/mpconfigvariant.props index ea8bf0bc82..d927994087 100644 --- a/ports/windows/variants/standard/mpconfigvariant.props +++ b/ports/windows/variants/standard/mpconfigvariant.props @@ -3,4 +3,7 @@ $(PyWinDir)\variants\manifest.py + + + From 0c5a399eb35e0dbd3194640cfca6818594080c20 Mon Sep 17 00:00:00 2001 From: stijn Date: Mon, 25 Mar 2024 14:38:08 +0100 Subject: [PATCH 7/9] windows/msvc: Introduce an msbuild variable for third-party sources. This is consistent with extmod.mk's split between SRC_EXTMOD_C and SRC_THIRDPARTY_C, the latter specifying source files which get compiled but not included in qstr generation. Signed-off-by: stijn --- ports/windows/micropython.vcxproj | 1 + ports/windows/msvc/genhdr.targets | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ports/windows/micropython.vcxproj b/ports/windows/micropython.vcxproj index 462821a6a0..bbdb0a1b7f 100644 --- a/ports/windows/micropython.vcxproj +++ b/ports/windows/micropython.vcxproj @@ -87,6 +87,7 @@ + diff --git a/ports/windows/msvc/genhdr.targets b/ports/windows/msvc/genhdr.targets index 7bd8364a9e..00afeed6f0 100644 --- a/ports/windows/msvc/genhdr.targets +++ b/ports/windows/msvc/genhdr.targets @@ -66,7 +66,7 @@ using(var outFile = System.IO.File.CreateText(OutputFile)) { - + False $([System.String]::new('%(FullPath)').Replace('$(PyBaseDir)', '$(DestDir)qstr\')) From 35e504f0613a4387986ce189fcf0b06da2f7b57f Mon Sep 17 00:00:00 2001 From: stijn Date: Mon, 25 Mar 2024 14:32:57 +0100 Subject: [PATCH 8/9] windows/msvc: Fix preprocessing command for quoted definitions. Supports preprocessor definitions like /Dval="quotedvalue" by turning that into /Dval=\"quotedvalue\". Signed-off-by: stijn --- ports/windows/msvc/genhdr.targets | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ports/windows/msvc/genhdr.targets b/ports/windows/msvc/genhdr.targets index 00afeed6f0..526b1a0a71 100644 --- a/ports/windows/msvc/genhdr.targets +++ b/ports/windows/msvc/genhdr.targets @@ -82,7 +82,9 @@ using(var outFile = System.IO.File.CreateText(OutputFile)) { - $(PyClTool) /nologo /I@(PyIncDirs, ' /I') /D@(PreProcDefs, ' /D') + + /D@(PreProcDefs, ' /D') + $(PyClTool) /nologo /I@(PyIncDirs, ' /I') $([System.String]::Copy('$(UnescapedPreProcDefs)').Replace('"','\"')) @(QstrDependencies->AnyHaveMetadataValue('Changed', 'True')) @(PyQstrSourceFiles->AnyHaveMetadataValue('Changed', 'True')) From dc44e7c43411c16f51494578449d66fc590ecff3 Mon Sep 17 00:00:00 2001 From: stijn Date: Wed, 25 Oct 2023 16:12:39 +0200 Subject: [PATCH 9/9] windows/msvc: Add ssl module using mbedtls in standard variant. Signed-off-by: stijn --- .../variants/standard/mpconfigvariant.h | 4 + .../variants/standard/mpconfigvariant.props | 82 +++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/ports/windows/variants/standard/mpconfigvariant.h b/ports/windows/variants/standard/mpconfigvariant.h index 3cdcfa8e9b..0e0f4cc0ae 100644 --- a/ports/windows/variants/standard/mpconfigvariant.h +++ b/ports/windows/variants/standard/mpconfigvariant.h @@ -26,3 +26,7 @@ #define MICROPY_PY_BUILTINS_HELP (1) #define MICROPY_PY_BUILTINS_HELP_MODULES (1) +#ifndef __MINGW32__ +#define MICROPY_PY_SSL (1) +#define MICROPY_SSL_MBEDTLS (1) +#endif diff --git a/ports/windows/variants/standard/mpconfigvariant.props b/ports/windows/variants/standard/mpconfigvariant.props index d927994087..2577b34631 100644 --- a/ports/windows/variants/standard/mpconfigvariant.props +++ b/ports/windows/variants/standard/mpconfigvariant.props @@ -2,8 +2,90 @@ $(PyWinDir)\variants\manifest.py + $(PyIncDirs);$(PyBaseDir)lib\mbedtls\include + + + MBEDTLS_CONFIG_FILE="$(PyBaseDir)ports\unix\mbedtls\mbedtls_config_port.h";%(PreprocessorDefinitions) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +