Merge branch 'micropython:master' into object_dict_len_test_ecw

.github/workflows/code_formatting.yml

@@ -10,7 +10,7 @@ jobs:
   code-formatting:
     runs-on: ubuntu-22.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: actions/setup-python@v4
     - name: Install packages
       run: source tools/ci.sh && ci_code_formatting_setup
@@ -22,7 +22,7 @@ jobs:
   code-spelling:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: actions/setup-python@v4
     - name: Install packages
       run: source tools/ci.sh && ci_code_spell_setup

.github/workflows/code_size.yml

@@ -20,7 +20,7 @@ jobs:
   build:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 100
     - name: Install packages

.github/workflows/commit_formatting.yml

@@ -10,7 +10,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: '100'
     - uses: actions/setup-python@v4

.github/workflows/docs.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: actions/setup-python@v4
     - name: Install Python packages
       run: pip install Sphinx

.github/workflows/examples.yml

@@ -18,7 +18,7 @@ jobs:
   embedding:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build
       run: make -C examples/embedding -f micropython_embed.mk && make -C examples/embedding
     - name: Run

.github/workflows/mpremote.yml

@@ -11,7 +11,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         # Setting this to zero means fetch all history and tags,
         # which hatch-vcs can use to discover the version tag.

.github/workflows/mpy_format.yml

@@ -17,7 +17,7 @@ jobs:
   test:
     runs-on: ubuntu-20.04 # use 20.04 to get python2
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install packages
       run: source tools/ci.sh && ci_mpy_format_setup
     - name: Test mpy-tool.py

.github/workflows/ports.yml

@@ -17,6 +17,6 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build ports download metadata
       run: mkdir boards && ./tools/autobuild/build-downloads.py . ./boards

.github/workflows/ports_cc3200.yml

@@ -21,7 +21,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install packages
       run: source tools/ci.sh && ci_cc3200_setup
     - name: Build

.github/workflows/ports_esp32.yml

@@ -21,7 +21,7 @@ jobs:
   build_idf50:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install packages
       run: source tools/ci.sh && ci_esp32_idf50_setup
     - name: Build

.github/workflows/ports_esp8266.yml

@@ -21,7 +21,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install packages
       run: source tools/ci.sh && ci_esp8266_setup && ci_esp8266_path >> $GITHUB_PATH
     - name: Build

.github/workflows/ports_mimxrt.yml

@@ -21,7 +21,7 @@ jobs:
   build:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install packages
       run: source tools/ci.sh && ci_mimxrt_setup
     - name: Build

.github/workflows/ports_nrf.yml

@@ -21,7 +21,7 @@ jobs:
   build:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install packages
       run: source tools/ci.sh && ci_nrf_setup
     - name: Build

.github/workflows/ports_powerpc.yml

@@ -21,7 +21,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install packages
       run: source tools/ci.sh && ci_powerpc_setup
     - name: Build

.github/workflows/ports_qemu-arm.yml

@@ -22,7 +22,7 @@ jobs:
   build_and_test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install packages
       run: source tools/ci.sh && ci_qemu_arm_setup
     - name: Build and run test suite

.github/workflows/ports_renesas-ra.yml

@@ -21,7 +21,7 @@ jobs:
   build_renesas_ra_board:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install packages
       run: source tools/ci.sh && ci_renesas_ra_setup
     - name: Build

.github/workflows/ports_rp2.yml

@@ -21,7 +21,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install packages
       run: source tools/ci.sh && ci_rp2_setup
     - name: Build

.github/workflows/ports_samd.yml

@@ -21,7 +21,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install packages
       run: source tools/ci.sh && ci_samd_setup
     - name: Build

.github/workflows/ports_stm32.yml

@@ -21,7 +21,7 @@ jobs:
   build_pyb:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install packages
       run: source tools/ci.sh && ci_stm32_setup
     - name: Build
@@ -30,7 +30,7 @@ jobs:
   build_nucleo:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install packages
       run: source tools/ci.sh && ci_stm32_setup
     - name: Build

.github/workflows/ports_teensy.yml

@@ -21,7 +21,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install packages
       run: source tools/ci.sh && ci_teensy_setup
     - name: Build

.github/workflows/ports_unix.yml

@@ -23,7 +23,7 @@ jobs:
   minimal:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build
       run: source tools/ci.sh && ci_unix_minimal_build
     - name: Run main test suite
@@ -35,7 +35,7 @@ jobs:
   reproducible:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build with reproducible date
       run: source tools/ci.sh && ci_unix_minimal_build
       env:
@@ -46,7 +46,7 @@ jobs:
   standard:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build
       run: source tools/ci.sh && ci_unix_standard_build
     - name: Run main test suite
@@ -58,7 +58,7 @@ jobs:
   coverage:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install packages
       run: source tools/ci.sh && ci_unix_coverage_setup
     - name: Build
@@ -87,7 +87,7 @@ jobs:
   coverage_32bit:
     runs-on: ubuntu-20.04 # use 20.04 to get libffi-dev:i386
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install packages
       run: source tools/ci.sh && ci_unix_32bit_setup
     - name: Build
@@ -105,7 +105,7 @@ jobs:
   nanbox:
     runs-on: ubuntu-20.04 # use 20.04 to get python2, and libffi-dev:i386
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install packages
       run: source tools/ci.sh && ci_unix_32bit_setup
     - name: Build
@@ -119,7 +119,7 @@ jobs:
   float:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build
       run: source tools/ci.sh && ci_unix_float_build
     - name: Run main test suite
@@ -131,7 +131,7 @@ jobs:
   stackless_clang:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install packages
       run: source tools/ci.sh && ci_unix_clang_setup
     - name: Build
@@ -145,7 +145,7 @@ jobs:
   float_clang:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install packages
       run: source tools/ci.sh && ci_unix_clang_setup
     - name: Build
@@ -159,7 +159,7 @@ jobs:
   settrace:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build
       run: source tools/ci.sh && ci_unix_settrace_build
     - name: Run main test suite
@@ -171,7 +171,7 @@ jobs:
   settrace_stackless:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build
       run: source tools/ci.sh && ci_unix_settrace_stackless_build
     - name: Run main test suite
@@ -183,7 +183,7 @@ jobs:
   macos:
     runs-on: macos-11.0
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: actions/setup-python@v4
       with:
         python-version: '3.8'
@@ -198,7 +198,7 @@ jobs:
   qemu_mips:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install packages
       run: source tools/ci.sh && ci_unix_qemu_mips_setup
     - name: Build
@@ -212,7 +212,7 @@ jobs:
   qemu_arm:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install packages
       run: source tools/ci.sh && ci_unix_qemu_arm_setup
     - name: Build

.github/workflows/ports_webassembly.yml

@@ -20,7 +20,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install packages
       run: source tools/ci.sh && ci_webassembly_setup
     - name: Build

.github/workflows/ports_windows.yml

@@ -21,7 +21,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install packages
       run: source tools/ci.sh && ci_windows_setup
     - name: Build

.github/workflows/ports_zephyr.yml

@@ -20,7 +20,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install packages
       run: source tools/ci.sh && ci_zephyr_setup
     - name: Install Zephyr

.github/workflows/ruff.yml

@@ -5,6 +5,6 @@ jobs:
   ruff:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - run: pip install --user ruff
     - run: ruff --format=github .

docs/develop/gettingstarted.rst

@@ -100,7 +100,7 @@ For the stm32 port, the ARM cross-compiler is required:
 
 .. code-block:: bash
 
-   $ sudo apt-get install arm-none-eabi-gcc arm-none-eabi-binutils arm-none-eabi-newlib
+   $ sudo apt-get install gcc-arm-none-eabi libnewlib-arm-none-eabi
 
 See the `ARM GCC
 toolchain <https://developer.arm.com/downloads/-/arm-gnu-toolchain-downloads>`_
@@ -228,7 +228,7 @@ You can also specify which board to use:
 .. code-block:: bash
 
    $ cd ports/stm32
-   $ make submodules
+   $ make BOARD=<board> submodules
    $ make BOARD=<board>
 
 See `ports/stm32/boards <https://github.com/micropython/micropython/tree/master/ports/stm32/boards>`_

docs/esp32/tutorial/peripheral_access.rst

@@ -42,3 +42,83 @@ for this:
 
 The MCPWM0 peripheral is in bit position 17 of the above two registers, hence
 the value of ``DPORT_PWM0_CLK_EN``.
+
+Synchronous access to pins directly via registers
+-------------------------------------------------
+
+The following code shows how to access pins directly via registers.  It has been
+tested on a generic ESP32 board.  It configures pins 16, 17, 32 and 33 in output
+mode via registers, and switches pin output values via registers.  Pins 16 and
+17 are switched simultaneously.
+
+.. code-block:: python3
+
+    from micropython import const
+    from machine import mem32, Pin
+
+    GPIO_OUT_REG = const(0x3FF44004)  # GPIO 0-31 output register
+    GPIO_OUT1_REG = const(0x3FF44010)  # GPIO 32-39 output register
+
+    GPIO_ENABLE_REG = const(0x3FF44020)  # GPIO 0-31 output enable register
+    GPIO_ENABLE1_REG = const(0x3FF4402C)  # GPIO 32-39 output enable register
+
+    M16 = 1 << 16  # Pin(16) bit mask
+    M17 = 1 << 17  # Pin(17) bit mask
+
+    M32 = 1 << (32-32)  # Pin(32) bit mask
+    M33 = 1 << (33-32)  # Pin(33) bit mask
+
+    # Enable pin output mode like
+    # p16 = Pin(16, mode=Pin.OUT)
+    # p17 = Pin(17, mode=Pin.OUT)
+    # p32 = Pin(32, mode=Pin.OUT)
+    # p33 = Pin(33, mode=Pin.OUT)
+    mem32[GPIO_ENABLE_REG] = mem32[GPIO_ENABLE_REG] | M16 | M17
+    mem32[GPIO_ENABLE1_REG] = mem32[GPIO_ENABLE1_REG] | M32 | M33
+
+    print(hex(mem32[GPIO_OUT_REG]), hex(mem32[GPIO_OUT1_REG]))
+
+    # Set outputs to 1 like
+    # p16(1)
+    # p17(1)
+    # p32(1)
+    # p33(1)
+    mem32[GPIO_OUT_REG] = mem32[GPIO_OUT_REG] | M16 | M17
+    mem32[GPIO_OUT1_REG] = mem32[GPIO_OUT1_REG] | M32 | M33
+
+    print(hex(mem32[GPIO_OUT_REG]), hex(mem32[GPIO_OUT1_REG]))
+
+    # Set outputs to 0 like
+    # p16(0)
+    # p17(0)
+    # p32(0)
+    # p33(0)
+    mem32[GPIO_OUT_REG] = mem32[GPIO_OUT_REG] & ~(M16 | M17)
+    mem32[GPIO_OUT1_REG] = mem32[GPIO_OUT1_REG] & ~(M32 | M33)
+
+    print(hex(mem32[GPIO_OUT_REG]), hex(mem32[GPIO_OUT1_REG]))
+
+    while True:
+        # Set outputs to 1
+        mem32[GPIO_OUT_REG] = mem32[GPIO_OUT_REG] | M16 | M17
+        mem32[GPIO_OUT1_REG] = mem32[GPIO_OUT1_REG] | M32 | M33
+
+        # Set outputs to 0
+        mem32[GPIO_OUT_REG] = mem32[GPIO_OUT_REG] & ~(M16 | M17)
+        mem32[GPIO_OUT1_REG] = mem32[GPIO_OUT1_REG] & ~(M32 | M33)
+
+
+Output is::
+
+    0x0 0x0
+    0x30000 0x3
+    0x0 0x0
+
+Pins 16 and 17 are switched synchronously:
+
+.. image:: img/mem32_gpio_output.jpg
+
+Same image on pins 32 and 33.
+
+Note that pins 34-36 and 39 are inputs only. Also pins 1 and 3 are Tx, Rx of the REPL UART,
+pins 6-11 are connected to the built-in SPI flash.

docs/library/deflate.rst

@@ -41,9 +41,15 @@ Classes
    to 1024 bytes. Valid values are ``5`` to ``15`` inclusive (corresponding to
    window sizes of 32 to 32k bytes).
 
-   If *wbits* is set to ``0`` (the default), then a window size of 256 bytes
-   will be used (corresponding to *wbits* set to ``8``), except when
-   :ref:`decompressing a zlib stream <deflate_wbits_zlib>`.
+   If *wbits* is set to ``0`` (the default), then for compression a window size
+   of 256 bytes will be used (as if *wbits* was set to 8). For decompression, it
+   depends on the format:
+
+   * ``RAW`` will use 256 bytes (corresponding to *wbits* set to 8).
+   * ``ZLIB`` (or ``AUTO`` with zlib detected) will use the value from the zlib
+     header.
+   * ``GZIP`` (or ``AUTO`` with gzip detected) will use 32 kilobytes
+     (corresponding to *wbits* set to 15).
 
    See the :ref:`window size <deflate_wbits>` notes below for more information
    about the window size, zlib, and gzip streams.
@@ -134,44 +140,43 @@ Deflate window size
 -------------------
 
 The window size limits how far back in the stream the (de)compressor can
-reference. Increasing the window size will improve compression, but will
-require more memory.
+reference. Increasing the window size will improve compression, but will require
+more memory and make the compressor slower.
 
-However, just because a given window size is used for compression, this does not
-mean that the stream will require the same size window for decompression, as
-the stream may not reference data as far back as the window allows (for example,
-if the length of the input is smaller than the window size).
+If an input stream was compressed a given window size, then `DeflateIO`
+using a smaller window size will fail mid-way during decompression with
+:exc:`OSError`, but only if a back-reference actually refers back further
+than the decompressor's window size. This means it may be possible to decompress
+with a smaller window size. For example, this would trivially be the case if the
+original uncompressed data is shorter than the window size.
 
-If the decompressor uses a smaller window size than necessary for the input data
-stream, it will fail mid-way through decompression with :exc:`OSError`.
+Decompression
+~~~~~~~~~~~~~
 
-.. _deflate_wbits_zlib:
+The zlib format includes a header which specifies the window size that was used
+to compress the data. This indicates the maximum window size required to
+decompress this stream. If this header value is less than the specified *wbits*
+value (or if *wbits* is unset), then the header value will be used.
 
-The zlib format includes a header which specifies the window size used to
-compress the data (which due to the above, may be larger than the size required
-for the decompressor).
+The gzip format does not include the window size in the header, and assumes that
+all gzip compressors (e.g. the ``gzip`` utility, or CPython's implementation of
+:class:`gzip.GzipFile`) use the maximum window size of 32kiB. For this reason,
+if the *wbits* parameter is not set, the decompressor will use a 32 kiB window
+size (corresponding to *wbits* set to 15). This means that to be able to
+decompress an arbitrary gzip stream, you must have at least this much RAM
+available. If you control the source data, consider instead using the zlib
+format with a smaller window size.
 
-If this header value is lower than the specified *wbits* value, then the header
-value will be used instead in order to reduce the memory allocation size. If
-the *wbits* parameter is zero (the default), then the header value will only be
-used if it is less than the maximum value of ``15`` (which is default value
-used by most compressors [#f1]_).
+The raw format has no header and therefore does not include any information
+about the window size. If *wbits* is not set, then it will default to a window
+size of 256 bytes, which may not be large enough for a given stream. Therefore
+it is recommended that you should always explicitly set *wbits* if using the raw
+format.
 
-In other words, if the source zlib stream has been compressed with a custom window
-size (i.e. less than ``15``), then using the default *wbits* parameter of zero
-will decompress any such stream.
+Compression
+~~~~~~~~~~~
 
-The gzip file format does not include the window size in the header.
-Additionally, most compressor libraries (including CPython's implementation
-of :class:`gzip.GzipFile`) will default to the maximum possible window size.
-This makes it difficult to decompress most gzip streams on MicroPython unless
-your board has a lot of free RAM.
-
-If you control the source of the compressed data, then prefer to use the zlib
-format, with a window size that is suitable for your target device.
-
-.. rubric:: Footnotes
-
-.. [#f1] The assumption here is that if the header value is the default used by
-   most compressors, then nothing is known about the likely required window
-   size and we should ignore it.
+For compression, MicroPython will default to a window size of 256 bytes for all
+formats. This provides a reasonable amount of compression with minimal memory
+usage and fast compression time, and will generate output that will work with
+any decompressor.

docs/library/esp32.rst

@@ -51,13 +51,20 @@ Functions
     buffers and other data. This data is useful to get a sense of how much memory
     is available to ESP-IDF and the networking stack in particular. It may shed
     some light on situations where ESP-IDF operations fail due to allocation failures.
-    The information returned is *not* useful to troubleshoot Python allocation failures,
-    use `micropython.mem_info()` instead.
 
     The capabilities parameter corresponds to ESP-IDF's ``MALLOC_CAP_XXX`` values but the
     two most useful ones are predefined as `esp32.HEAP_DATA` for data heap regions and
     `esp32.HEAP_EXEC` for executable regions as used by the native code emitter.
 
+    Free IDF heap memory in the `esp32.HEAP_DATA` region is available to be
+    automatically added to the MicroPython heap to prevent a MicroPython
+    allocation from failing. However, the information returned here is otherwise
+    *not* useful to troubleshoot Python allocation failures, use
+    `micropython.mem_info()` instead. The "max new split" value in
+    `micropython.mem_info()` output corresponds to the largest free block of
+    ESP-IDF heap that could be automatically added on demand to the MicroPython
+    heap.
+
     The return value is a list of 4-tuples, where each 4-tuple corresponds to one heap
     and contains: the total bytes, the free bytes, the largest free block, and
     the minimum free seen over time.
@@ -107,6 +114,11 @@ methods to enable over-the-air (OTA) updates.
 
     Sets the partition as the boot partition.
 
+    .. note:: Do not enter :func:`deepsleep<machine.deepsleep>` after changing
+       the OTA boot partition, without first performing a hard
+       :func:`reset<machine.reset>` or power cycle. This ensures the bootloader
+       will validate the new image before booting.
+
 .. method:: Partition.get_next_update()
 
     Gets the next update partition after this one, and returns a new Partition object.

docs/library/index.rst

@@ -71,6 +71,7 @@ library.
    json.rst
    math.rst
    os.rst
+   platform.rst
    random.rst
    re.rst
    select.rst

docs/library/neopixel.rst

@@ -9,9 +9,7 @@ This module provides a driver for WS2818 / NeoPixel LEDs.
 .. note:: This module is only included by default on the ESP8266, ESP32 and RP2
    ports. On STM32 / Pyboard and others, you can either install the
    ``neopixel`` package using :term:`mip`, or you can download the module
-   directly from
-   <https://raw.githubusercontent.com/micropython/micropython-lib/master/micropython/drivers/led/neopixel/neopixel.py>`_
-   and copy it to the filesystem.
+   directly from :term:`micropython-lib` and copy it to the filesystem.
 
 class NeoPixel
 --------------

docs/library/network.rst

@@ -171,8 +171,8 @@ The following are functions available in the network module.
 
 .. function:: hostname([name])
 
-    Get or set the hostname that will identify this device on the network. It is
-    applied to all interfaces.
+    Get or set the hostname that will identify this device on the network. It will
+    be used by all interfaces.
 
     This hostname is used for:
      * Sending to the DHCP server in the client request. (If using DHCP)
@@ -182,6 +182,12 @@ The following are functions available in the network module.
     If the function is called without parameters, it returns the current
     hostname.
 
+    A change in hostname is typically only applied during connection. For DHCP
+    this is because the hostname is part of the DHCP client request, and the
+    implementation of mDNS in most ports only initialises the hostname once
+    during connection. For this reason, you must set the hostname before
+    activating/connecting your network interfaces.
+
     The default hostname is typically the name of the board.
 
 .. function:: phy_mode([mode])

docs/library/platform.rst

@@ -0,0 +1,38 @@
+:mod:`platform` -- access to underlying platform’s identifying data
+===================================================================
+
+.. module:: platform
+   :synopsis: access to underlying platform’s identifying data
+
+|see_cpython_module| :mod:`python:platform`.
+
+This module tries to retrieve as much platform-identifying data as possible. It
+makes this information available via function APIs.
+
+Functions
+---------
+
+.. function:: platform()
+
+   Returns a string identifying the underlying platform. This string is composed
+   of several substrings in the following order, delimited by dashes (``-``):
+
+   - the name of the platform system (e.g. Unix, Windows or MicroPython)
+   - the MicroPython version
+   - the architecture of the platform
+   - the version of the underlying platform
+   - the concatenation of the name of the libc that MicroPython is linked to
+     and its corresponding version.
+
+   For example, this could be
+   ``"MicroPython-1.20.0-xtensa-IDFv4.2.4-with-newlib3.0.0"``.
+
+.. function:: python_compiler()
+
+   Returns a string identifying the compiler used for compiling MicroPython.
+
+.. function:: libc_ver()
+
+   Returns a tuple of strings *(lib, version)*, where *lib* is the name of the
+   libc that MicroPython is linked to, and *version* the corresponding version
+   of this libc.

docs/library/stm.rst

@@ -102,3 +102,39 @@ the second CPU, the RF core.
     Execute a HCI command on the SYS channel.  The execution is synchronous.
 
     Returns a bytes object with the result of the SYS command.
+
+Functions specific to STM32WLxx MCUs
+------------------------------------
+
+These functions are available on STM32WLxx microcontrollers, and interact with
+the integrated "SUBGHZ" radio modem peripheral.
+
+.. function:: subghz_cs(level)
+
+   Sets the internal SPI CS pin attached to the radio peripheral. The ``level``
+   argument is active-low: a truthy value means "CS pin high" and de-asserts the
+   signal, a falsey value means "CS pin low" and asserts the signal.
+
+   The internal-only SPI bus corresponding to this CS signal can be instantiated
+   using :ref:`machine.SPI()<machine.SPI>` ``id`` value ``"SUBGHZ"``.
+
+.. function:: subghz_irq(handler)
+
+   Sets the internal SUBGHZ radio interrupt handler to the provided
+   function. The handler function is called as a "hard" interrupt in response to
+   radio peripheral interrupts. See :ref:`isr_rules` for more information about
+   interrupt handlers in MicroPython.
+
+   Calling this function with the handler argument set to None disables the IRQ.
+
+   Due to a hardware limitation, each time this IRQ fires MicroPython disables
+   it before calling the handler. In order to receive another interrupt, Python
+   code should call ``subghz_irq()`` to set the handler again. This has the side
+   effect of re-enabling the IRQ.
+
+.. function:: subghz_is_busy()
+
+   Return a ``bool`` corresponding to the internal "RFBUSYS" signal from the
+   radio peripheral. Before sending a new command to the radio over SPI then
+   this function should be polled until it returns ``False``, to confirm the
+   busy signal is de-asserted.

drivers/cyw43/cywbt.c

@@ -29,8 +29,6 @@
 
 #include "py/runtime.h"
 #include "py/mphal.h"
-#include "pin_static_af.h"
-#include "uart.h"
 #include "extmod/mpbthci.h"
 
 #if MICROPY_PY_NETWORK_CYW43
@@ -38,33 +36,41 @@
 #include "lib/cyw43-driver/src/cyw43_config.h"
 #include "lib/cyw43-driver/firmware/cyw43_btfw_4343A1.h"
 
-// Provided by the port.
-extern pyb_uart_obj_t mp_bluetooth_hci_uart_obj;
-
 // Provided by the port, and also possibly shared with the stack.
 extern uint8_t mp_bluetooth_hci_cmd_buf[4 + 256];
 
 /******************************************************************************/
 // CYW BT HCI low-level driver
 
+#ifdef CYW43_PIN_BT_CTS
+// This code is not portable and currently only builds on stm32 port.
+
+#include "pin_static_af.h"
+#include "uart.h"
+
+// Provided by the port.
+extern pyb_uart_obj_t mp_bluetooth_hci_uart_obj;
+
 STATIC void cywbt_wait_cts_low(void) {
-    mp_hal_pin_config(pyb_pin_BT_CTS, MP_HAL_PIN_MODE_INPUT, MP_HAL_PIN_PULL_UP, 0);
+    mp_hal_pin_config(CYW43_PIN_BT_CTS, MP_HAL_PIN_MODE_INPUT, MP_HAL_PIN_PULL_UP, 0);
     for (int i = 0; i < 200; ++i) {
-        if (mp_hal_pin_read(pyb_pin_BT_CTS) == 0) {
+        if (mp_hal_pin_read(CYW43_PIN_BT_CTS) == 0) {
             break;
         }
         mp_hal_delay_ms(1);
     }
-    mp_hal_pin_config_alt(pyb_pin_BT_CTS, MP_HAL_PIN_MODE_ALT, MP_HAL_PIN_PULL_UP, AF_FN_UART, mp_bluetooth_hci_uart_obj.uart_id);
+    mp_hal_pin_config_alt(CYW43_PIN_BT_CTS, MP_HAL_PIN_MODE_ALT,
+        MP_HAL_PIN_PULL_UP, AF_FN_UART, mp_bluetooth_hci_uart_obj.uart_id);
 }
+#endif
 
 STATIC int cywbt_hci_cmd_raw(size_t len, uint8_t *buf) {
-    uart_tx_strn(&mp_bluetooth_hci_uart_obj, (void*)buf, len);
-    for (int i = 0; i < 6; ++i) {
-        while (!uart_rx_any(&mp_bluetooth_hci_uart_obj)) {
+    mp_bluetooth_hci_uart_write((void *)buf, len);
+    for (int c, i = 0; i < 6; ++i) {
+        while ((c = mp_bluetooth_hci_uart_readchar()) == -1) {
             MICROPY_EVENT_POLL_HOOK
         }
-        buf[i] = uart_rx_char(&mp_bluetooth_hci_uart_obj);
+        buf[i] = c;
     }
 
     // expect a command complete event (event 0x0e)
@@ -80,11 +86,11 @@ STATIC int cywbt_hci_cmd_raw(size_t len, uint8_t *buf) {
         */
 
     int sz = buf[2] - 3;
-    for (int i = 0; i < sz; ++i) {
-        while (!uart_rx_any(&mp_bluetooth_hci_uart_obj)) {
+    for (int c, i = 0; i < sz; ++i) {
+        while ((c = mp_bluetooth_hci_uart_readchar()) == -1) {
             MICROPY_EVENT_POLL_HOOK
         }
-        buf[i] = uart_rx_char(&mp_bluetooth_hci_uart_obj);
+        buf[i] = c;
     }
 
     return 0;
@@ -150,12 +156,15 @@ STATIC int cywbt_download_firmware(const uint8_t *firmware) {
 
     // RF switch must select high path during BT patch boot
     #if MICROPY_HW_ENABLE_RF_SWITCH
-    mp_hal_pin_config(pyb_pin_WL_GPIO_1, MP_HAL_PIN_MODE_INPUT, MP_HAL_PIN_PULL_UP, 0);
+    mp_hal_pin_config(CYW43_PIN_WL_GPIO_1, MP_HAL_PIN_MODE_INPUT, MP_HAL_PIN_PULL_UP, 0);
     #endif
     mp_hal_delay_ms(10); // give some time for CTS to go high
+    #ifdef CYW43_PIN_BT_CTS
     cywbt_wait_cts_low();
+    #endif
     #if MICROPY_HW_ENABLE_RF_SWITCH
-    mp_hal_pin_config(pyb_pin_WL_GPIO_1, MP_HAL_PIN_MODE_INPUT, MP_HAL_PIN_PULL_DOWN, 0); // Select chip antenna (could also select external)
+    // Select chip antenna (could also select external)
+    mp_hal_pin_config(CYW43_PIN_WL_GPIO_1, MP_HAL_PIN_MODE_INPUT, MP_HAL_PIN_PULL_DOWN, 0);
     #endif
 
     mp_bluetooth_hci_uart_set_baudrate(115200);
@@ -168,25 +177,33 @@ STATIC int cywbt_download_firmware(const uint8_t *firmware) {
 int mp_bluetooth_hci_controller_init(void) {
     // This is called immediately after the UART is initialised during stack initialisation.
 
-    mp_hal_pin_output(pyb_pin_BT_REG_ON);
-    mp_hal_pin_low(pyb_pin_BT_REG_ON);
-    mp_hal_pin_input(pyb_pin_BT_HOST_WAKE);
-    mp_hal_pin_output(pyb_pin_BT_DEV_WAKE);
-    mp_hal_pin_low(pyb_pin_BT_DEV_WAKE);
+    mp_hal_pin_output(CYW43_PIN_BT_REG_ON);
+    mp_hal_pin_low(CYW43_PIN_BT_REG_ON);
+    #ifdef CYW43_PIN_BT_HOST_WAKE
+    mp_hal_pin_input(CYW43_PIN_BT_HOST_WAKE);
+    #endif
+    #ifdef CYW43_PIN_BT_DEV_WAKE
+    mp_hal_pin_output(CYW43_PIN_BT_DEV_WAKE);
+    mp_hal_pin_low(CYW43_PIN_BT_DEV_WAKE);
+    #endif
 
     #if MICROPY_HW_ENABLE_RF_SWITCH
     // TODO don't select antenna if wifi is enabled
-    mp_hal_pin_config(pyb_pin_WL_GPIO_4, MP_HAL_PIN_MODE_OUTPUT, MP_HAL_PIN_PULL_NONE, 0); // RF-switch power
-    mp_hal_pin_high(pyb_pin_WL_GPIO_4); // Turn the RF-switch on
+    mp_hal_pin_config(CYW43_PIN_WL_GPIO_4, MP_HAL_PIN_MODE_OUTPUT, MP_HAL_PIN_PULL_NONE, 0); // RF-switch power
+    mp_hal_pin_high(CYW43_PIN_WL_GPIO_4); // Turn the RF-switch on
     #endif
 
     uint8_t buf[256];
 
-    mp_hal_pin_low(pyb_pin_BT_REG_ON);
+    mp_hal_pin_low(CYW43_PIN_BT_REG_ON);
     mp_bluetooth_hci_uart_set_baudrate(115200);
     mp_hal_delay_ms(100);
-    mp_hal_pin_high(pyb_pin_BT_REG_ON);
+    mp_hal_pin_high(CYW43_PIN_BT_REG_ON);
+    #ifdef CYW43_PIN_BT_CTS
     cywbt_wait_cts_low();
+    #else
+    mp_hal_delay_ms(100);
+    #endif
 
     // Reset
     cywbt_hci_cmd(0x03, 0x0003, 0, NULL);
@@ -197,7 +214,7 @@ int mp_bluetooth_hci_controller_init(void) {
     mp_bluetooth_hci_uart_set_baudrate(MICROPY_HW_BLE_UART_BAUDRATE_DOWNLOAD_FIRMWARE);
     #endif
 
-    cywbt_download_firmware((const uint8_t*)&cyw43_btfw_4343A1[0]);
+    cywbt_download_firmware((const uint8_t *)&cyw43_btfw_4343A1[0]);
 
     // Reset
     cywbt_hci_cmd(0x03, 0x0003, 0, NULL);
@@ -219,31 +236,33 @@ int mp_bluetooth_hci_controller_init(void) {
     // cywbt_hci_cmd(0x03, 0x0013, 248, buf);
 
     // Configure sleep mode
-    cywbt_hci_cmd(0x3f, 0x27, 12, (const uint8_t*)"\x01\x02\x02\x00\x00\x00\x01\x00\x00\x00\x00\x00");
+    cywbt_hci_cmd(0x3f, 0x27, 12, (const uint8_t *)"\x01\x02\x02\x00\x00\x00\x01\x00\x00\x00\x00\x00");
 
     // HCI_Write_LE_Host_Support
-    cywbt_hci_cmd(3, 109, 2, (const uint8_t*)"\x01\x00");
+    cywbt_hci_cmd(3, 109, 2, (const uint8_t *)"\x01\x00");
 
-    mp_hal_pin_high(pyb_pin_BT_DEV_WAKE); // let sleep
+    #ifdef CYW43_PIN_BT_DEV_WAKE
+    mp_hal_pin_high(CYW43_PIN_BT_DEV_WAKE); // let sleep
+    #endif
 
     return 0;
 }
 
 int mp_bluetooth_hci_controller_deinit(void) {
-    mp_hal_pin_low(pyb_pin_BT_REG_ON);
+    mp_hal_pin_low(CYW43_PIN_BT_REG_ON);
 
     return 0;
 }
 
-#ifdef pyb_pin_BT_DEV_WAKE
+#ifdef CYW43_PIN_BT_DEV_WAKE
 STATIC uint32_t bt_sleep_ticks;
 #endif
 
 int mp_bluetooth_hci_controller_sleep_maybe(void) {
-    #ifdef pyb_pin_BT_DEV_WAKE
-    if (mp_hal_pin_read(pyb_pin_BT_DEV_WAKE) == 0) {
+    #ifdef CYW43_PIN_BT_DEV_WAKE
+    if (mp_hal_pin_read(CYW43_PIN_BT_DEV_WAKE) == 0) {
         if (mp_hal_ticks_ms() - bt_sleep_ticks > 500) {
-            mp_hal_pin_high(pyb_pin_BT_DEV_WAKE); // let sleep
+            mp_hal_pin_high(CYW43_PIN_BT_DEV_WAKE); // let sleep
         }
     }
     #endif
@@ -251,8 +270,8 @@ int mp_bluetooth_hci_controller_sleep_maybe(void) {
 }
 
 bool mp_bluetooth_hci_controller_woken(void) {
-    #ifdef pyb_pin_BT_HOST_WAKE
-    bool host_wake = mp_hal_pin_read(pyb_pin_BT_HOST_WAKE);
+    #ifdef CYW43_PIN_BT_HOST_WAKE
+    bool host_wake = mp_hal_pin_read(CYW43_PIN_BT_HOST_WAKE);
     /*
     // this is just for info/tracing purposes
     static bool last_host_wake = false;
@@ -268,11 +287,11 @@ bool mp_bluetooth_hci_controller_woken(void) {
 }
 
 int mp_bluetooth_hci_controller_wakeup(void) {
-    #ifdef pyb_pin_BT_DEV_WAKE
+    #ifdef CYW43_PIN_BT_DEV_WAKE
     bt_sleep_ticks = mp_hal_ticks_ms();
 
-    if (mp_hal_pin_read(pyb_pin_BT_DEV_WAKE) == 1) {
-        mp_hal_pin_low(pyb_pin_BT_DEV_WAKE); // wake up
+    if (mp_hal_pin_read(CYW43_PIN_BT_DEV_WAKE) == 1) {
+        mp_hal_pin_low(CYW43_PIN_BT_DEV_WAKE); // wake up
         // Use delay_us rather than delay_ms to prevent running the scheduler (which
         // might result in more BLE operations).
         mp_hal_delay_us(5000); // can't go lower than this

examples/asmled.py

@@ -1,5 +1,9 @@
 # flash LED #1 using inline assembler
 # this version is overly verbose and uses word stores
+#
+# ruff: noqa: F821 - @asm_thumb decorator adds names to function scope
+
+
 @micropython.asm_thumb
 def flash_led(r0):
     movw(r1, (stm.GPIOA + stm.GPIO_BSRRL) & 0xFFFF)

examples/asmsum.py

@@ -1,3 +1,6 @@
+# ruff: noqa: F821 - @asm_thumb decorator adds names to function scope
+
+
 @micropython.asm_thumb
 def asm_sum_words(r0, r1):
     # r0 = len

examples/bluetooth/ble_advertising.py

@@ -1,5 +1,9 @@
 # Helpers for generating BLE advertising payloads.
 
+# A more fully-featured (and easier to use) version of this is implemented in
+# aioble. This code is provided just as a basic example. See
+# https://github.com/micropython/micropython-lib/tree/master/micropython/bluetooth/aioble
+
 from micropython import const
 import struct
 import bluetooth
@@ -19,6 +23,8 @@ _ADV_TYPE_UUID32_MORE = const(0x4)
 _ADV_TYPE_UUID128_MORE = const(0x6)
 _ADV_TYPE_APPEARANCE = const(0x19)
 
+_ADV_MAX_PAYLOAD = const(31)
+
 
 # Generate a payload to be passed to gap_advertise(adv_data=...).
 def advertising_payload(limited_disc=False, br_edr=False, name=None, services=None, appearance=0):
@@ -50,6 +56,9 @@ def advertising_payload(limited_disc=False, br_edr=False, name=None, services=No
     if appearance:
         _append(_ADV_TYPE_APPEARANCE, struct.pack("<h", appearance))
 
+    if len(payload) > _ADV_MAX_PAYLOAD:
+        raise ValueError("advertising payload too large")
+
     return payload
 
 

examples/bluetooth/ble_bonding_peripheral.py

@@ -4,6 +4,11 @@
 # any connected central every 10 seconds.
 #
 # Work-in-progress demo of implementing bonding and passkey auth.
+#
+# This example demonstrates the low-level bluetooth module. For most
+# applications, we recommend using the higher-level aioble library, which
+# includes an implementation of the secret store. See
+# https://github.com/micropython/micropython-lib/tree/master/micropython/bluetooth/aioble
 
 import bluetooth
 import random

examples/bluetooth/ble_simple_central.py

@@ -1,6 +1,11 @@
 # This example finds and connects to a peripheral running the
 # UART service (e.g. ble_simple_peripheral.py).
 
+# This example demonstrates the low-level bluetooth module. For most
+# applications, we recommend using the higher-level aioble library which takes
+# care of all IRQ handling and connection management. See
+# https://github.com/micropython/micropython-lib/tree/master/micropython/bluetooth/aioble
+
 import bluetooth
 import random
 import struct

examples/bluetooth/ble_simple_peripheral.py

@@ -1,5 +1,10 @@
 # This example demonstrates a UART periperhal.
 
+# This example demonstrates the low-level bluetooth module. For most
+# applications, we recommend using the higher-level aioble library which takes
+# care of all IRQ handling and connection management. See
+# https://github.com/micropython/micropython-lib/tree/master/micropython/bluetooth/aioble
+
 import bluetooth
 import random
 import struct

examples/bluetooth/ble_temperature.py

@@ -3,6 +3,12 @@
 # The sensor's local value updates every second, and it will notify
 # any connected central every 10 seconds.
 
+# This example demonstrates the low-level bluetooth module. For most
+# applications, we recommend using the higher-level aioble library which takes
+# care of all IRQ handling and connection management. See
+# https://github.com/micropython/micropython-lib/tree/master/micropython/bluetooth/aioble
+# and in particular the temp_sensor.py example included with aioble.
+
 import bluetooth
 import random
 import struct

examples/bluetooth/ble_temperature_central.py

@@ -1,5 +1,11 @@
 # This example finds and connects to a BLE temperature sensor (e.g. the one in ble_temperature.py).
 
+# This example demonstrates the low-level bluetooth module. For most
+# applications, we recommend using the higher-level aioble library which takes
+# care of all IRQ handling and connection management. See
+# https://github.com/micropython/micropython-lib/tree/master/micropython/bluetooth/aioble
+# and in particular the temp_client.py example included with aioble.
+
 import bluetooth
 import random
 import struct

examples/bluetooth/ble_uart_peripheral.py

@@ -1,5 +1,10 @@
 # This example demonstrates a peripheral implementing the Nordic UART Service (NUS).
 
+# This example demonstrates the low-level bluetooth module. For most
+# applications, we recommend using the higher-level aioble library which takes
+# care of all IRQ handling and connection management. See
+# https://github.com/micropython/micropython-lib/tree/master/micropython/bluetooth/aioble
+
 import bluetooth
 from ble_advertising import advertising_payload
 

examples/bluetooth/ble_uart_repl.py

@@ -7,6 +7,7 @@ import bluetooth
 import io
 import os
 import micropython
+from micropython import const
 import machine
 
 from ble_uart_peripheral import BLEUART

examples/hwapi/hwconfig_z_96b_carbon.py

@@ -1,4 +1,4 @@
-from machine import Signal
+from machine import Pin, Signal
 
 # 96Boards Carbon board
 # USR1 - User controlled led, connected to PD2

examples/natmod/README.md

@@ -0,0 +1,74 @@
+# Dynamic Native Modules
+
+Dynamic Native Modules are .mpy files that contain native machine code from a
+language other than Python. For more info see [the documentation]
+(https://docs.micropython.org/en/latest/develop/natmod.html).
+
+This should not be confused with [User C Modules]
+(https://docs.micropython.org/en/latest/develop/cmodules.html) which are a
+mechanism to add additional out-of-tree modules into the firmware build.
+
+## Examples
+
+This directory contains several examples of writing dynamic native modules, in
+two main categories:
+
+1.  Feature examples.
+
+    * `features0` - A module containing a single "factorial" function which
+      demonstrates working with integers.
+
+    * `features1` - A module that demonstrates some common tasks:
+        - defining simple functions exposed to Python
+        - defining local, helper C functions
+        - defining constant integers and strings exposed to Python
+        - getting and creating integer objects
+        - creating Python lists
+        - raising exceptions
+        - allocating memory
+        - BSS and constant data (rodata)
+        - relocated pointers in rodata
+
+    * `features2` - This is a hybrid module containing both Python and C code,
+      and additionally the C code is spread over multiple files. It also
+      demonstrates using floating point (only when the target supports
+      hardware floating point).
+
+    * `features3` - A module that shows how to use types, constant objects,
+      and creating dictionary instances.
+
+    * `features4` - A module that demonstrates how to define a class.
+
+2.  Dynamic version of existing built-ins.
+
+    This provides a way to add missing functionality to firmware that doesn't
+    include certain built-in modules. See the `heapq`, `random`, `re`,
+    `deflate`, `btree`, and `framebuf` directories.
+
+    So for example, if your firmware was compiled with `MICROPY_PY_FRAMEBUF`
+    disabled (e.g. to save flash space), then it would not include the
+    `framebuf` module. The `framebuf` native module provides a way to add the
+    `framebuf` module dynamically.
+
+    The way these work is they define a dynamic native module which
+    `#include`'s the original module and then does the necessary
+    initialisation of the module's globals dict.
+
+## Build instructions
+
+To compile an example, you need to have the same toolchain available as
+required for your target port. e.g. `arm-none-eabi-gcc` for any ARM Cortex M
+target. See the port instructions for details.
+
+You also need to have the `pyelftools` Python package available, either via
+your system package manager or installed from PyPI in a virtual environment
+with `pip`.
+
+Each example provides a Makefile. You should specify the `ARCH` argument to
+make (one of x86, x64, armv6m, armv7m, xtensa, xtensawin):
+
+```
+$ cd features0
+$ make ARCH=armv7m
+$ mpremote cp features0.mpy :
+```

examples/natmod/btree/btree_py.py

@@ -1,3 +1,7 @@
 # Implemented in Python to support keyword arguments
+
+# ruff: noqa: F821 - this file is evaluated with C-defined names in scope
+
+
 def open(stream, *, flags=0, cachesize=0, pagesize=0, minkeypage=0):
     return _open(stream, flags, cachesize, pagesize, minkeypage)

examples/natmod/features2/test.py

@@ -1,5 +1,7 @@
 # This Python code will be merged with the C code in main.c
 
+# ruff: noqa: F821 - this file is evaluated with C-defined names in scope
+
 import array
 
 

examples/natmod/features4/Makefile

@@ -0,0 +1,14 @@
+# Location of top-level MicroPython directory
+MPY_DIR = ../../..
+
+# Name of module
+MOD = features4
+
+# Source files (.c or .py)
+SRC = features4.c
+
+# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin)
+ARCH = x64
+
+# Include to get the rules for compiling and linking the module
+include $(MPY_DIR)/py/dynruntime.mk

examples/natmod/features4/features4.c

@@ -0,0 +1,73 @@
+/*
+  This example extends on features0 but demonstrates how to define a class.
+
+  The Factorial class constructor takes an integer, and then the calculate
+  method can be called to get the factorial.
+
+  >>> import features4
+  >>> f = features4.Factorial(4)
+  >>> f.calculate()
+  24
+*/
+
+// Include the header file to get access to the MicroPython API
+#include "py/dynruntime.h"
+
+// This is type(Factorial)
+mp_obj_full_type_t mp_type_factorial;
+
+// This is the internal state of a Factorial instance.
+typedef struct {
+    mp_obj_base_t base;
+    mp_int_t n;
+} mp_obj_factorial_t;
+
+// Essentially Factorial.__new__ (but also kind of __init__).
+// Takes a single argument (the number to find the factorial of)
+STATIC mp_obj_t factorial_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args_in) {
+    mp_arg_check_num(n_args, n_kw, 1, 1, false);
+
+    mp_obj_factorial_t *o = mp_obj_malloc(mp_obj_factorial_t, type);
+    o->n = mp_obj_get_int(args_in[0]);
+
+    return MP_OBJ_FROM_PTR(o);
+}
+
+STATIC mp_int_t factorial_helper(mp_int_t x) {
+    if (x == 0) {
+        return 1;
+    }
+    return x * factorial_helper(x - 1);
+}
+
+// Implements Factorial.calculate()
+STATIC mp_obj_t factorial_calculate(mp_obj_t self_in) {
+    mp_obj_factorial_t *self = MP_OBJ_TO_PTR(self_in);
+    return mp_obj_new_int(factorial_helper(self->n));
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_1(factorial_calculate_obj, factorial_calculate);
+
+// Locals dict for the Factorial type (will have a single method, calculate,
+// added in mpy_init).
+mp_map_elem_t factorial_locals_dict_table[1];
+STATIC MP_DEFINE_CONST_DICT(factorial_locals_dict, factorial_locals_dict_table);
+
+// This is the entry point and is called when the module is imported
+mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+    // This must be first, it sets up the globals dict and other things
+    MP_DYNRUNTIME_INIT_ENTRY
+
+    // Initialise the type.
+    mp_type_factorial.base.type = (void*)&mp_type_type;
+    mp_type_factorial.flags = MP_TYPE_FLAG_NONE;
+    mp_type_factorial.name = MP_QSTR_Factorial;
+    MP_OBJ_TYPE_SET_SLOT(&mp_type_factorial, make_new, factorial_make_new, 0);
+    factorial_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_calculate), MP_OBJ_FROM_PTR(&factorial_calculate_obj) };
+    MP_OBJ_TYPE_SET_SLOT(&mp_type_factorial, locals_dict, (void*)&factorial_locals_dict, 1);
+
+    // Make the Factorial type available on the module.
+    mp_store_global(MP_QSTR_Factorial, MP_OBJ_FROM_PTR(&mp_type_factorial));
+
+    // This must be last, it restores the globals dict
+    MP_DYNRUNTIME_INIT_EXIT
+}

examples/rp2/pio_1hz.py

@@ -1,6 +1,8 @@
 # Example using PIO to blink an LED and raise an IRQ at 1Hz.
 # Note: this does not work on Pico W because it uses Pin(25) for LED output.
 
+# ruff: noqa: F821 - @asm_pio decorator adds names to function scope
+
 import time
 from machine import Pin
 import rp2

examples/rp2/pio_exec.py

@@ -5,6 +5,8 @@
 #   - using set_init and set_base
 #   - using StateMachine.exec
 
+# ruff: noqa: F821 - @asm_pio decorator adds names to function scope
+
 import time
 from machine import Pin
 import rp2

examples/rp2/pio_pinchange.py

@@ -8,6 +8,8 @@
 #   - setting an irq handler for a StateMachine
 #   - instantiating 2x StateMachine's with the same program and different pins
 
+# ruff: noqa: F821 - @asm_pio decorator adds names to function scope
+
 import time
 from machine import Pin
 import rp2

examples/rp2/pio_pwm.py

@@ -1,5 +1,7 @@
 # Example of using PIO for PWM, and fading the brightness of an LED
 
+# ruff: noqa: F821 - @asm_pio decorator adds names to function scope
+
 from machine import Pin
 from rp2 import PIO, StateMachine, asm_pio
 from time import sleep

examples/rp2/pio_uart_rx.py

@@ -8,6 +8,8 @@
 #   - PIO irq handler
 #   - using the second core via _thread
 
+# ruff: noqa: F821 - @asm_pio decorator adds names to function scope
+
 import _thread
 from machine import Pin, UART
 from rp2 import PIO, StateMachine, asm_pio

examples/rp2/pio_uart_tx.py

@@ -1,5 +1,7 @@
 # Example using PIO to create a UART TX interface
 
+# ruff: noqa: F821 - @asm_pio decorator adds names to function scope
+
 from machine import Pin
 from rp2 import PIO, StateMachine, asm_pio
 

examples/rp2/pio_ws2812.py

@@ -1,5 +1,7 @@
 # Example using PIO to drive a set of WS2812 LEDs.
 
+# ruff: noqa: F821 - @asm_pio decorator adds names to function scope
+
 import array, time
 from machine import Pin
 import rp2

examples/rp2/pwm_fade.py

@@ -1,6 +1,8 @@
 # Example using PWM to fade an LED.
 # Note: this does not work on Pico W because it uses Pin(25) for LED output.
 
+# ruff: noqa: F821 - @asm_pio decorator adds names to function scope
+
 import time
 from machine import Pin, PWM
 

extmod/extmod.cmake

@@ -60,51 +60,62 @@ set(MICROPY_SOURCE_EXTMOD
 
 if(MICROPY_PY_BTREE)
     set(MICROPY_LIB_BERKELEY_DIR "${MICROPY_DIR}/lib/berkeley-db-1.xx")
+    string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/berkeley-db-1.xx)
 
-    add_library(micropy_extmod_btree OBJECT
-        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_close.c
-        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_conv.c
-        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_debug.c
-        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_delete.c
-        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_get.c
-        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_open.c
-        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_overflow.c
-        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_page.c
-        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_put.c
-        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_search.c
-        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_seq.c
-        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_split.c
-        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_utils.c
-        ${MICROPY_LIB_BERKELEY_DIR}/mpool/mpool.c
-    )
+    if(ECHO_SUBMODULES)
+        # No-op, we're just doing submodule/variant discovery.
+        # Cannot run the add_library/target_include_directories rules (even though
+        # the build won't run) because IDF will attempt verify the files exist.
+    elseif(NOT EXISTS ${MICROPY_LIB_BERKELEY_DIR}/README)
+        # Regular build, submodule not initialised -- fail with a clear error.
+        message(FATAL_ERROR " MICROPY_PY_BTREE is enabled but the berkeley-db submodule is not initialised.\n Run 'make BOARD=${MICROPY_BOARD} submodules'")
+    else()
+        # Regular build, we have the submodule.
+        add_library(micropy_extmod_btree OBJECT
+            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_close.c
+            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_conv.c
+            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_debug.c
+            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_delete.c
+            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_get.c
+            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_open.c
+            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_overflow.c
+            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_page.c
+            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_put.c
+            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_search.c
+            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_seq.c
+            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_split.c
+            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_utils.c
+            ${MICROPY_LIB_BERKELEY_DIR}/mpool/mpool.c
+        )
 
-    target_include_directories(micropy_extmod_btree PRIVATE
-        ${MICROPY_LIB_BERKELEY_DIR}/PORT/include
-    )
+        target_include_directories(micropy_extmod_btree PRIVATE
+            ${MICROPY_LIB_BERKELEY_DIR}/PORT/include
+        )
 
-    target_compile_definitions(micropy_extmod_btree PRIVATE
-        __DBINTERFACE_PRIVATE=1
-        mpool_error=printf
-        abort=abort_
-        "virt_fd_t=void*"
-    )
+        target_compile_definitions(micropy_extmod_btree PRIVATE
+            __DBINTERFACE_PRIVATE=1
+            mpool_error=printf
+            abort=abort_
+            "virt_fd_t=void*"
+        )
 
-    # The include directories and compile definitions below are needed to build
-    # modbtree.c and should be added to the main MicroPython target.
+        # The include directories and compile definitions below are needed to build
+        # modbtree.c and should be added to the main MicroPython target.
 
-    list(APPEND MICROPY_INC_CORE
-        "${MICROPY_LIB_BERKELEY_DIR}/PORT/include"
-    )
+        list(APPEND MICROPY_INC_CORE
+            "${MICROPY_LIB_BERKELEY_DIR}/PORT/include"
+        )
 
-    list(APPEND MICROPY_DEF_CORE
-        MICROPY_PY_BTREE=1
-        __DBINTERFACE_PRIVATE=1
-        "virt_fd_t=void*"
-    )
+        list(APPEND MICROPY_DEF_CORE
+            MICROPY_PY_BTREE=1
+            __DBINTERFACE_PRIVATE=1
+            "virt_fd_t=void*"
+        )
 
-    list(APPEND MICROPY_SOURCE_EXTMOD
-        ${MICROPY_EXTMOD_DIR}/modbtree.c
-    )
+        list(APPEND MICROPY_SOURCE_EXTMOD
+            ${MICROPY_EXTMOD_DIR}/modbtree.c
+        )
+    endif()
 endif()
 
 # Library for mbedtls

extmod/moddeflate.c

@@ -54,6 +54,8 @@ typedef enum {
     DEFLATEIO_FORMAT_MAX = DEFLATEIO_FORMAT_GZIP,
 } deflateio_format_t;
 
+// This is used when the wbits is unset in the DeflateIO constructor. Default
+// to the smallest window size (faster compression, less RAM usage, etc).
 const int DEFLATEIO_DEFAULT_WBITS = 8;
 
 typedef struct {
@@ -114,24 +116,32 @@ STATIC bool deflateio_init_read(mp_obj_deflateio_t *self) {
     // Don't modify self->window_bits as it may also be used for write.
     int wbits = self->window_bits;
 
-    // Parse the header if we're in NONE/ZLIB/GZIP modes.
-    if (self->format != DEFLATEIO_FORMAT_RAW) {
-        int header_wbits = wbits;
+    if (self->format == DEFLATEIO_FORMAT_RAW) {
+        if (wbits == 0) {
+            // The docs recommends always setting wbits explicitly when using
+            // RAW, but we still allow a default.
+            wbits = DEFLATEIO_DEFAULT_WBITS;
+        }
+    } else {
+        // Parse the header if we're in NONE/ZLIB/GZIP modes.
+        int header_wbits;
         int header_type = uzlib_parse_zlib_gzip_header(&self->read->decomp, &header_wbits);
-        if ((self->format == DEFLATEIO_FORMAT_ZLIB && header_type != UZLIB_HEADER_ZLIB) || (self->format == DEFLATEIO_FORMAT_GZIP && header_type != UZLIB_HEADER_GZIP)) {
+        if (header_type < 0) {
+            // Stream header was invalid.
             return false;
         }
-        if (wbits == 0 && header_wbits < 15) {
-            // If the header specified something lower than the default, then
-            // use that instead.
+        if ((self->format == DEFLATEIO_FORMAT_ZLIB && header_type != UZLIB_HEADER_ZLIB) || (self->format == DEFLATEIO_FORMAT_GZIP && header_type != UZLIB_HEADER_GZIP)) {
+            // Not what we expected.
+            return false;
+        }
+        // header_wbits will either be 15 (gzip) or 8-15 (zlib).
+        if (wbits == 0 || header_wbits < wbits) {
+            // If the header specified something lower, then use that instead.
+            // No point doing a bigger allocation than we need to.
             wbits = header_wbits;
         }
     }
 
-    if (wbits == 0) {
-        wbits = DEFLATEIO_DEFAULT_WBITS;
-    }
-
     size_t window_len = 1 << wbits;
     self->read->window = m_new(uint8_t, window_len);
 
@@ -163,6 +173,7 @@ STATIC bool deflateio_init_write(mp_obj_deflateio_t *self) {
 
     int wbits = self->window_bits;
     if (wbits == 0) {
+        // Same default wbits for all formats.
         wbits = DEFLATEIO_DEFAULT_WBITS;
     }
     size_t window_len = 1 << wbits;

extmod/modlwip.c

@@ -1377,7 +1377,8 @@ STATIC mp_obj_t lwip_socket_setsockopt(size_t n_args, const mp_obj_t *args) {
 
     switch (opt) {
         // level: SOL_SOCKET
-        case SOF_REUSEADDR: {
+        case SOF_REUSEADDR:
+        case SOF_BROADCAST: {
             mp_int_t val = mp_obj_get_int(args[3]);
             // Options are common for UDP and TCP pcb's.
             if (val) {
@@ -1786,6 +1787,7 @@ STATIC const mp_rom_map_elem_t mp_module_lwip_globals_table[] = {
 
     { MP_ROM_QSTR(MP_QSTR_SOL_SOCKET), MP_ROM_INT(1) },
     { MP_ROM_QSTR(MP_QSTR_SO_REUSEADDR), MP_ROM_INT(SOF_REUSEADDR) },
+    { MP_ROM_QSTR(MP_QSTR_SO_BROADCAST), MP_ROM_INT(SOF_BROADCAST) },
 
     { MP_ROM_QSTR(MP_QSTR_IPPROTO_IP), MP_ROM_INT(0) },
     { MP_ROM_QSTR(MP_QSTR_IP_ADD_MEMBERSHIP), MP_ROM_INT(IP_ADD_MEMBERSHIP) },

extmod/modnetwork.h

@@ -43,6 +43,7 @@
 
 // Common option flags per-socket.
 #define MOD_NETWORK_SO_REUSEADDR    (0x0004)
+#define MOD_NETWORK_SO_BROADCAST    (0x0020)
 #define MOD_NETWORK_SO_KEEPALIVE    (0x0008)
 #define MOD_NETWORK_SO_SNDTIMEO     (0x1005)
 #define MOD_NETWORK_SO_RCVTIMEO     (0x1006)

extmod/modselect.c

@@ -3,7 +3,7 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2014 Damien P. George
+ * Copyright (c) 2014-2023 Damien P. George
  * Copyright (c) 2015-2017 Paul Sokolovsky
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -26,10 +26,6 @@
  */
 
 #include "py/mpconfig.h"
-#if MICROPY_PY_SELECT
-
-#include <stdio.h>
-
 #include "py/runtime.h"
 #include "py/obj.h"
 #include "py/objlist.h"
@@ -37,51 +33,248 @@
 #include "py/mperrno.h"
 #include "py/mphal.h"
 
-// Flags for poll()
+#if MICROPY_PY_SELECT
+
+#if MICROPY_PY_SELECT_SELECT && MICROPY_PY_SELECT_POSIX_OPTIMISATIONS
+#error "select.select is not supported with MICROPY_PY_SELECT_POSIX_OPTIMISATIONS"
+#endif
+
+#if MICROPY_PY_SELECT_POSIX_OPTIMISATIONS
+
+#include <poll.h>
+
+#if !((MP_STREAM_POLL_RD) == (POLLIN) && \
+    (MP_STREAM_POLL_WR) == (POLLOUT) && \
+    (MP_STREAM_POLL_ERR) == (POLLERR) && \
+    (MP_STREAM_POLL_HUP) == (POLLHUP) && \
+    (MP_STREAM_POLL_NVAL) == (POLLNVAL))
+#error "With MICROPY_PY_SELECT_POSIX_OPTIMISATIONS enabled, POLL constants must match"
+#endif
+
+// When non-file-descriptor objects are on the list to be polled (the polling of
+// which involves repeatedly calling ioctl(MP_STREAM_POLL)), this variable sets
+// the period between polling these objects.
+#define MICROPY_PY_SELECT_IOCTL_CALL_PERIOD_MS (1)
+
+#endif
+
+// Flags for ipoll()
 #define FLAG_ONESHOT (1)
 
+// A single pollable object.
 typedef struct _poll_obj_t {
     mp_obj_t obj;
     mp_uint_t (*ioctl)(mp_obj_t obj, mp_uint_t request, uintptr_t arg, int *errcode);
-    mp_uint_t flags;
-    mp_uint_t flags_ret;
+    #if MICROPY_PY_SELECT_POSIX_OPTIMISATIONS
+    // If the pollable object has an associated file descriptor, then pollfd points to an entry
+    // in poll_set_t::pollfds, and the events/revents fields for this object are stored in the
+    // pollfd entry (and the nonfd_* members are unused).
+    // Otherwise the object is a non-file-descriptor object and pollfd==NULL, and the events/
+    // revents fields are stored in the nonfd_* members (which are named as such so that code
+    // doesn't accidentally mix the use of these members when this optimisation is used).
+    struct pollfd *pollfd;
+    uint16_t nonfd_events;
+    uint16_t nonfd_revents;
+    #else
+    mp_uint_t events;
+    mp_uint_t revents;
+    #endif
 } poll_obj_t;
 
-STATIC void poll_map_add(mp_map_t *poll_map, const mp_obj_t *obj, mp_uint_t obj_len, mp_uint_t flags, bool or_flags) {
+// A set of pollable objects.
+typedef struct _poll_set_t {
+    // Map containing a dict with key=object to poll, value=its corresponding poll_obj_t.
+    mp_map_t map;
+
+    #if MICROPY_PY_SELECT_POSIX_OPTIMISATIONS
+    // Array of pollfd entries for objects that have a file descriptor.
+    unsigned short alloc; // memory allocated for pollfds
+    unsigned short max_used; // maximum number of used entries in pollfds
+    unsigned short used; // actual number of used entries in pollfds
+    struct pollfd *pollfds;
+    #endif
+} poll_set_t;
+
+STATIC void poll_set_init(poll_set_t *poll_set, size_t n) {
+    mp_map_init(&poll_set->map, n);
+    #if MICROPY_PY_SELECT_POSIX_OPTIMISATIONS
+    poll_set->alloc = 0;
+    poll_set->max_used = 0;
+    poll_set->used = 0;
+    poll_set->pollfds = NULL;
+    #endif
+}
+
+#if MICROPY_PY_SELECT_SELECT
+STATIC void poll_set_deinit(poll_set_t *poll_set) {
+    mp_map_deinit(&poll_set->map);
+}
+#endif
+
+#if MICROPY_PY_SELECT_POSIX_OPTIMISATIONS
+
+STATIC mp_uint_t poll_obj_get_events(poll_obj_t *poll_obj) {
+    assert(poll_obj->pollfd == NULL);
+    return poll_obj->nonfd_events;
+}
+
+STATIC void poll_obj_set_events(poll_obj_t *poll_obj, mp_uint_t events) {
+    if (poll_obj->pollfd != NULL) {
+        poll_obj->pollfd->events = events;
+    } else {
+        poll_obj->nonfd_events = events;
+    }
+}
+
+STATIC mp_uint_t poll_obj_get_revents(poll_obj_t *poll_obj) {
+    if (poll_obj->pollfd != NULL) {
+        return poll_obj->pollfd->revents;
+    } else {
+        return poll_obj->nonfd_revents;
+    }
+}
+
+STATIC void poll_obj_set_revents(poll_obj_t *poll_obj, mp_uint_t revents) {
+    if (poll_obj->pollfd != NULL) {
+        poll_obj->pollfd->revents = revents;
+    } else {
+        poll_obj->nonfd_revents = revents;
+    }
+}
+
+STATIC struct pollfd *poll_set_add_fd(poll_set_t *poll_set, int fd) {
+    struct pollfd *free_slot = NULL;
+
+    if (poll_set->used == poll_set->max_used) {
+        // No free slots below max_used, so expand max_used (and possibly allocate).
+        if (poll_set->max_used >= poll_set->alloc) {
+            poll_set->pollfds = m_renew(struct pollfd, poll_set->pollfds, poll_set->alloc, poll_set->alloc + 4);
+            poll_set->alloc += 4;
+        }
+        free_slot = &poll_set->pollfds[poll_set->max_used++];
+    } else {
+        // There should be a free slot below max_used.
+        for (unsigned int i = 0; i < poll_set->max_used; ++i) {
+            struct pollfd *slot = &poll_set->pollfds[i];
+            if (slot->fd == -1) {
+                free_slot = slot;
+                break;
+            }
+        }
+        assert(free_slot != NULL);
+    }
+
+    free_slot->fd = fd;
+    ++poll_set->used;
+
+    return free_slot;
+}
+
+static inline bool poll_set_all_are_fds(poll_set_t *poll_set) {
+    return poll_set->map.used == poll_set->used;
+}
+
+#else
+
+static inline mp_uint_t poll_obj_get_events(poll_obj_t *poll_obj) {
+    return poll_obj->events;
+}
+
+static inline void poll_obj_set_events(poll_obj_t *poll_obj, mp_uint_t events) {
+    poll_obj->events = events;
+}
+
+static inline mp_uint_t poll_obj_get_revents(poll_obj_t *poll_obj) {
+    return poll_obj->revents;
+}
+
+static inline void poll_obj_set_revents(poll_obj_t *poll_obj, mp_uint_t revents) {
+    poll_obj->revents = revents;
+}
+
+#endif
+
+STATIC void poll_set_add_obj(poll_set_t *poll_set, const mp_obj_t *obj, mp_uint_t obj_len, mp_uint_t events, bool or_events) {
     for (mp_uint_t i = 0; i < obj_len; i++) {
-        mp_map_elem_t *elem = mp_map_lookup(poll_map, mp_obj_id(obj[i]), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND);
+        mp_map_elem_t *elem = mp_map_lookup(&poll_set->map, mp_obj_id(obj[i]), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND);
         if (elem->value == MP_OBJ_NULL) {
             // object not found; get its ioctl and add it to the poll list
-            const mp_stream_p_t *stream_p = mp_get_stream_raise(obj[i], MP_STREAM_OP_IOCTL);
+
+            // If an exception is raised below when adding the new object then the map entry for that
+            // object remains unpopulated, and methods like poll() may crash.  This case is not handled.
+
             poll_obj_t *poll_obj = m_new_obj(poll_obj_t);
             poll_obj->obj = obj[i];
+
+            #if MICROPY_PY_SELECT_POSIX_OPTIMISATIONS
+            int fd = -1;
+            if (mp_obj_is_int(obj[i])) {
+                // A file descriptor integer passed in as the object, so use it directly.
+                fd = mp_obj_get_int(obj[i]);
+                if (fd < 0) {
+                    mp_raise_ValueError(NULL);
+                }
+                poll_obj->ioctl = NULL;
+            } else {
+                // An object passed in.  Check if it has a file descriptor.
+                const mp_stream_p_t *stream_p = mp_get_stream_raise(obj[i], MP_STREAM_OP_IOCTL);
+                poll_obj->ioctl = stream_p->ioctl;
+                int err;
+                mp_uint_t res = stream_p->ioctl(obj[i], MP_STREAM_GET_FILENO, 0, &err);
+                if (res != MP_STREAM_ERROR) {
+                    fd = res;
+                }
+            }
+            if (fd >= 0) {
+                // Object has a file descriptor so add it to pollfds.
+                poll_obj->pollfd = poll_set_add_fd(poll_set, fd);
+            } else {
+                // Object doesn't have a file descriptor.
+                poll_obj->pollfd = NULL;
+            }
+            #else
+            const mp_stream_p_t *stream_p = mp_get_stream_raise(obj[i], MP_STREAM_OP_IOCTL);
             poll_obj->ioctl = stream_p->ioctl;
-            poll_obj->flags = flags;
-            poll_obj->flags_ret = 0;
+            #endif
+
+            poll_obj_set_events(poll_obj, events);
+            poll_obj_set_revents(poll_obj, 0);
             elem->value = MP_OBJ_FROM_PTR(poll_obj);
         } else {
-            // object exists; update its flags
-            if (or_flags) {
-                ((poll_obj_t *)MP_OBJ_TO_PTR(elem->value))->flags |= flags;
-            } else {
-                ((poll_obj_t *)MP_OBJ_TO_PTR(elem->value))->flags = flags;
+            // object exists; update its events
+            poll_obj_t *poll_obj = (poll_obj_t *)MP_OBJ_TO_PTR(elem->value);
+            #if MICROPY_PY_SELECT_SELECT
+            if (or_events) {
+                events |= poll_obj_get_events(poll_obj);
             }
+            #else
+            (void)or_events;
+            #endif
+            poll_obj_set_events(poll_obj, events);
         }
     }
 }
 
-// poll each object in the map
-STATIC mp_uint_t poll_map_poll(mp_map_t *poll_map, size_t *rwx_num) {
+// For each object in the poll set, poll it once.
+STATIC mp_uint_t poll_set_poll_once(poll_set_t *poll_set, size_t *rwx_num) {
     mp_uint_t n_ready = 0;
-    for (mp_uint_t i = 0; i < poll_map->alloc; ++i) {
-        if (!mp_map_slot_is_filled(poll_map, i)) {
+    for (mp_uint_t i = 0; i < poll_set->map.alloc; ++i) {
+        if (!mp_map_slot_is_filled(&poll_set->map, i)) {
             continue;
         }
 
-        poll_obj_t *poll_obj = MP_OBJ_TO_PTR(poll_map->table[i].value);
+        poll_obj_t *poll_obj = MP_OBJ_TO_PTR(poll_set->map.table[i].value);
+
+        #if MICROPY_PY_SELECT_POSIX_OPTIMISATIONS
+        if (poll_obj->pollfd != NULL) {
+            // Object has file descriptor so will be polled separately by poll().
+            continue;
+        }
+        #endif
+
         int errcode;
-        mp_int_t ret = poll_obj->ioctl(poll_obj->obj, MP_STREAM_POLL, poll_obj->flags, &errcode);
-        poll_obj->flags_ret = ret;
+        mp_int_t ret = poll_obj->ioctl(poll_obj->obj, MP_STREAM_POLL, poll_obj_get_events(poll_obj), &errcode);
+        poll_obj_set_revents(poll_obj, ret);
 
         if (ret == -1) {
             // error doing ioctl
@@ -91,6 +284,7 @@ STATIC mp_uint_t poll_map_poll(mp_map_t *poll_map, size_t *rwx_num) {
         if (ret != 0) {
             // object is ready
             n_ready += 1;
+            #if MICROPY_PY_SELECT_SELECT
             if (rwx_num != NULL) {
                 if (ret & MP_STREAM_POLL_RD) {
                     rwx_num[0] += 1;
@@ -102,11 +296,82 @@ STATIC mp_uint_t poll_map_poll(mp_map_t *poll_map, size_t *rwx_num) {
                     rwx_num[2] += 1;
                 }
             }
+            #else
+            (void)rwx_num;
+            #endif
         }
     }
     return n_ready;
 }
 
+STATIC mp_uint_t poll_set_poll_until_ready_or_timeout(poll_set_t *poll_set, size_t *rwx_num, mp_uint_t timeout) {
+    mp_uint_t start_ticks = mp_hal_ticks_ms();
+
+    #if MICROPY_PY_SELECT_POSIX_OPTIMISATIONS
+
+    for (;;) {
+        MP_THREAD_GIL_EXIT();
+
+        // Compute the timeout.
+        int t = MICROPY_PY_SELECT_IOCTL_CALL_PERIOD_MS;
+        if (poll_set_all_are_fds(poll_set)) {
+            // All our pollables are file descriptors, so we can use a blocking
+            // poll and let it (the underlying system) handle the timeout.
+            if (timeout == (mp_uint_t)-1) {
+                t = -1;
+            } else {
+                mp_uint_t delta = mp_hal_ticks_ms() - start_ticks;
+                if (delta >= timeout) {
+                    t = 0;
+                } else {
+                    t = timeout - delta;
+                }
+            }
+        }
+
+        // Call system poll for those objects that have a file descriptor.
+        int n_ready = poll(poll_set->pollfds, poll_set->max_used, t);
+
+        MP_THREAD_GIL_ENTER();
+
+        // The call to poll() may have been interrupted, but per PEP 475 we must retry if the
+        // signal is EINTR (this implements a special case of calling MP_HAL_RETRY_SYSCALL()).
+        if (n_ready == -1) {
+            int err = errno;
+            if (err != EINTR) {
+                mp_raise_OSError(err);
+            }
+            n_ready = 0;
+        }
+
+        // Explicitly poll any objects that do not have a file descriptor.
+        if (!poll_set_all_are_fds(poll_set)) {
+            n_ready += poll_set_poll_once(poll_set, rwx_num);
+        }
+
+        // Return if an object is ready, or if the timeout expired.
+        if (n_ready > 0 || (timeout != (mp_uint_t)-1 && mp_hal_ticks_ms() - start_ticks >= timeout)) {
+            return n_ready;
+        }
+
+        // This would be MICROPY_EVENT_POLL_HOOK but the call to poll() above already includes a delay.
+        mp_handle_pending(true);
+    }
+
+    #else
+
+    for (;;) {
+        // poll the objects
+        mp_uint_t n_ready = poll_set_poll_once(poll_set, rwx_num);
+        if (n_ready > 0 || (timeout != (mp_uint_t)-1 && mp_hal_ticks_ms() - start_ticks >= timeout)) {
+            return n_ready;
+        }
+        MICROPY_EVENT_POLL_HOOK
+    }
+
+    #endif
+}
+
 #if MICROPY_PY_SELECT_SELECT
 // select(rlist, wlist, xlist[, timeout])
 STATIC mp_obj_t select_select(size_t n_args, const mp_obj_t *args) {
@@ -133,52 +398,46 @@ STATIC mp_obj_t select_select(size_t n_args, const mp_obj_t *args) {
     }
 
     // merge separate lists and get the ioctl function for each object
-    mp_map_t poll_map;
-    mp_map_init(&poll_map, rwx_len[0] + rwx_len[1] + rwx_len[2]);
-    poll_map_add(&poll_map, r_array, rwx_len[0], MP_STREAM_POLL_RD, true);
-    poll_map_add(&poll_map, w_array, rwx_len[1], MP_STREAM_POLL_WR, true);
-    poll_map_add(&poll_map, x_array, rwx_len[2], MP_STREAM_POLL_ERR | MP_STREAM_POLL_HUP, true);
+    poll_set_t poll_set;
+    poll_set_init(&poll_set, rwx_len[0] + rwx_len[1] + rwx_len[2]);
+    poll_set_add_obj(&poll_set, r_array, rwx_len[0], MP_STREAM_POLL_RD, true);
+    poll_set_add_obj(&poll_set, w_array, rwx_len[1], MP_STREAM_POLL_WR, true);
+    poll_set_add_obj(&poll_set, x_array, rwx_len[2], MP_STREAM_POLL_ERR | MP_STREAM_POLL_HUP, true);
 
-    mp_uint_t start_tick = mp_hal_ticks_ms();
+    // poll all objects
     rwx_len[0] = rwx_len[1] = rwx_len[2] = 0;
-    for (;;) {
-        // poll the objects
-        mp_uint_t n_ready = poll_map_poll(&poll_map, rwx_len);
+    poll_set_poll_until_ready_or_timeout(&poll_set, rwx_len, timeout);
 
-        if (n_ready > 0 || (timeout != (mp_uint_t)-1 && mp_hal_ticks_ms() - start_tick >= timeout)) {
-            // one or more objects are ready, or we had a timeout
-            mp_obj_t list_array[3];
-            list_array[0] = mp_obj_new_list(rwx_len[0], NULL);
-            list_array[1] = mp_obj_new_list(rwx_len[1], NULL);
-            list_array[2] = mp_obj_new_list(rwx_len[2], NULL);
-            rwx_len[0] = rwx_len[1] = rwx_len[2] = 0;
-            for (mp_uint_t i = 0; i < poll_map.alloc; ++i) {
-                if (!mp_map_slot_is_filled(&poll_map, i)) {
-                    continue;
-                }
-                poll_obj_t *poll_obj = MP_OBJ_TO_PTR(poll_map.table[i].value);
-                if (poll_obj->flags_ret & MP_STREAM_POLL_RD) {
-                    ((mp_obj_list_t *)MP_OBJ_TO_PTR(list_array[0]))->items[rwx_len[0]++] = poll_obj->obj;
-                }
-                if (poll_obj->flags_ret & MP_STREAM_POLL_WR) {
-                    ((mp_obj_list_t *)MP_OBJ_TO_PTR(list_array[1]))->items[rwx_len[1]++] = poll_obj->obj;
-                }
-                if ((poll_obj->flags_ret & ~(MP_STREAM_POLL_RD | MP_STREAM_POLL_WR)) != 0) {
-                    ((mp_obj_list_t *)MP_OBJ_TO_PTR(list_array[2]))->items[rwx_len[2]++] = poll_obj->obj;
-                }
-            }
-            mp_map_deinit(&poll_map);
-            return mp_obj_new_tuple(3, list_array);
+    // one or more objects are ready, or we had a timeout
+    mp_obj_t list_array[3];
+    list_array[0] = mp_obj_new_list(rwx_len[0], NULL);
+    list_array[1] = mp_obj_new_list(rwx_len[1], NULL);
+    list_array[2] = mp_obj_new_list(rwx_len[2], NULL);
+    rwx_len[0] = rwx_len[1] = rwx_len[2] = 0;
+    for (mp_uint_t i = 0; i < poll_set.map.alloc; ++i) {
+        if (!mp_map_slot_is_filled(&poll_set.map, i)) {
+            continue;
+        }
+        poll_obj_t *poll_obj = MP_OBJ_TO_PTR(poll_set.map.table[i].value);
+        if (poll_obj->revents & MP_STREAM_POLL_RD) {
+            ((mp_obj_list_t *)MP_OBJ_TO_PTR(list_array[0]))->items[rwx_len[0]++] = poll_obj->obj;
+        }
+        if (poll_obj->revents & MP_STREAM_POLL_WR) {
+            ((mp_obj_list_t *)MP_OBJ_TO_PTR(list_array[1]))->items[rwx_len[1]++] = poll_obj->obj;
+        }
+        if ((poll_obj->revents & ~(MP_STREAM_POLL_RD | MP_STREAM_POLL_WR)) != 0) {
+            ((mp_obj_list_t *)MP_OBJ_TO_PTR(list_array[2]))->items[rwx_len[2]++] = poll_obj->obj;
         }
-        MICROPY_EVENT_POLL_HOOK
     }
+    poll_set_deinit(&poll_set);
+    return mp_obj_new_tuple(3, list_array);
 }
 MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_select_select_obj, 3, 4, select_select);
 #endif // MICROPY_PY_SELECT_SELECT
 
 typedef struct _mp_obj_poll_t {
     mp_obj_base_t base;
-    mp_map_t poll_map;
+    poll_set_t poll_set;
     short iter_cnt;
     short iter_idx;
     int flags;
@@ -189,13 +448,13 @@ typedef struct _mp_obj_poll_t {
 // register(obj[, eventmask])
 STATIC mp_obj_t poll_register(size_t n_args, const mp_obj_t *args) {
     mp_obj_poll_t *self = MP_OBJ_TO_PTR(args[0]);
-    mp_uint_t flags;
+    mp_uint_t events;
     if (n_args == 3) {
-        flags = mp_obj_get_int(args[2]);
+        events = mp_obj_get_int(args[2]);
     } else {
-        flags = MP_STREAM_POLL_RD | MP_STREAM_POLL_WR;
+        events = MP_STREAM_POLL_RD | MP_STREAM_POLL_WR;
     }
-    poll_map_add(&self->poll_map, &args[1], 1, flags, false);
+    poll_set_add_obj(&self->poll_set, &args[1], 1, events, false);
     return mp_const_none;
 }
 MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(poll_register_obj, 2, 3, poll_register);
@@ -203,7 +462,21 @@ MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(poll_register_obj, 2, 3, poll_register);
 // unregister(obj)
 STATIC mp_obj_t poll_unregister(mp_obj_t self_in, mp_obj_t obj_in) {
     mp_obj_poll_t *self = MP_OBJ_TO_PTR(self_in);
-    mp_map_lookup(&self->poll_map, mp_obj_id(obj_in), MP_MAP_LOOKUP_REMOVE_IF_FOUND);
+    mp_map_elem_t *elem = mp_map_lookup(&self->poll_set.map, mp_obj_id(obj_in), MP_MAP_LOOKUP_REMOVE_IF_FOUND);
+
+    #if MICROPY_PY_SELECT_POSIX_OPTIMISATIONS
+    if (elem != NULL) {
+        poll_obj_t *poll_obj = (poll_obj_t *)MP_OBJ_TO_PTR(elem->value);
+        if (poll_obj->pollfd != NULL) {
+            poll_obj->pollfd->fd = -1;
+            --self->poll_set.used;
+        }
+        elem->value = MP_OBJ_NULL;
+    }
+    #else
+    (void)elem;
+    #endif
+
     // TODO raise KeyError if obj didn't exist in map
     return mp_const_none;
 }
@@ -212,11 +485,11 @@ MP_DEFINE_CONST_FUN_OBJ_2(poll_unregister_obj, poll_unregister);
 // modify(obj, eventmask)
 STATIC mp_obj_t poll_modify(mp_obj_t self_in, mp_obj_t obj_in, mp_obj_t eventmask_in) {
     mp_obj_poll_t *self = MP_OBJ_TO_PTR(self_in);
-    mp_map_elem_t *elem = mp_map_lookup(&self->poll_map, mp_obj_id(obj_in), MP_MAP_LOOKUP);
+    mp_map_elem_t *elem = mp_map_lookup(&self->poll_set.map, mp_obj_id(obj_in), MP_MAP_LOOKUP);
     if (elem == NULL) {
         mp_raise_OSError(MP_ENOENT);
     }
-    ((poll_obj_t *)MP_OBJ_TO_PTR(elem->value))->flags = mp_obj_get_int(eventmask_in);
+    poll_obj_set_events((poll_obj_t *)MP_OBJ_TO_PTR(elem->value), mp_obj_get_int(eventmask_in));
     return mp_const_none;
 }
 MP_DEFINE_CONST_FUN_OBJ_3(poll_modify_obj, poll_modify);
@@ -241,18 +514,7 @@ STATIC mp_uint_t poll_poll_internal(uint n_args, const mp_obj_t *args) {
 
     self->flags = flags;
 
-    mp_uint_t start_tick = mp_hal_ticks_ms();
-    mp_uint_t n_ready;
-    for (;;) {
-        // poll the objects
-        n_ready = poll_map_poll(&self->poll_map, NULL);
-        if (n_ready > 0 || (timeout != (mp_uint_t)-1 && mp_hal_ticks_ms() - start_tick >= timeout)) {
-            break;
-        }
-        MICROPY_EVENT_POLL_HOOK
-    }
-
-    return n_ready;
+    return poll_set_poll_until_ready_or_timeout(&self->poll_set, NULL, timeout);
 }
 
 STATIC mp_obj_t poll_poll(size_t n_args, const mp_obj_t *args) {
@@ -262,23 +524,19 @@ STATIC mp_obj_t poll_poll(size_t n_args, const mp_obj_t *args) {
     // one or more objects are ready, or we had a timeout
     mp_obj_list_t *ret_list = MP_OBJ_TO_PTR(mp_obj_new_list(n_ready, NULL));
     n_ready = 0;
-    for (mp_uint_t i = 0; i < self->poll_map.alloc; ++i) {
-        if (!mp_map_slot_is_filled(&self->poll_map, i)) {
+    for (mp_uint_t i = 0; i < self->poll_set.map.alloc; ++i) {
+        if (!mp_map_slot_is_filled(&self->poll_set.map, i)) {
             continue;
         }
-        poll_obj_t *poll_obj = MP_OBJ_TO_PTR(self->poll_map.table[i].value);
-        if (poll_obj->flags_ret != 0) {
-            mp_obj_t tuple[2] = {poll_obj->obj, MP_OBJ_NEW_SMALL_INT(poll_obj->flags_ret)};
+        poll_obj_t *poll_obj = MP_OBJ_TO_PTR(self->poll_set.map.table[i].value);
+        if (poll_obj_get_revents(poll_obj) != 0) {
+            mp_obj_t tuple[2] = {poll_obj->obj, MP_OBJ_NEW_SMALL_INT(poll_obj_get_revents(poll_obj))};
             ret_list->items[n_ready++] = mp_obj_new_tuple(2, tuple);
-            if (self->flags & FLAG_ONESHOT) {
-                // Don't poll next time, until new event flags will be set explicitly
-                poll_obj->flags = 0;
-            }
         }
     }
     return MP_OBJ_FROM_PTR(ret_list);
 }
-MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(poll_poll_obj, 1, 3, poll_poll);
+MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(poll_poll_obj, 1, 2, poll_poll);
 
 STATIC mp_obj_t poll_ipoll(size_t n_args, const mp_obj_t *args) {
     mp_obj_poll_t *self = MP_OBJ_TO_PTR(args[0]);
@@ -304,19 +562,19 @@ STATIC mp_obj_t poll_iternext(mp_obj_t self_in) {
 
     self->iter_cnt--;
 
-    for (mp_uint_t i = self->iter_idx; i < self->poll_map.alloc; ++i) {
+    for (mp_uint_t i = self->iter_idx; i < self->poll_set.map.alloc; ++i) {
         self->iter_idx++;
-        if (!mp_map_slot_is_filled(&self->poll_map, i)) {
+        if (!mp_map_slot_is_filled(&self->poll_set.map, i)) {
             continue;
         }
-        poll_obj_t *poll_obj = MP_OBJ_TO_PTR(self->poll_map.table[i].value);
-        if (poll_obj->flags_ret != 0) {
+        poll_obj_t *poll_obj = MP_OBJ_TO_PTR(self->poll_set.map.table[i].value);
+        if (poll_obj_get_revents(poll_obj) != 0) {
             mp_obj_tuple_t *t = MP_OBJ_TO_PTR(self->ret_tuple);
             t->items[0] = poll_obj->obj;
-            t->items[1] = MP_OBJ_NEW_SMALL_INT(poll_obj->flags_ret);
+            t->items[1] = MP_OBJ_NEW_SMALL_INT(poll_obj_get_revents(poll_obj));
             if (self->flags & FLAG_ONESHOT) {
-                // Don't poll next time, until new event flags will be set explicitly
-                poll_obj->flags = 0;
+                // Don't poll next time, until new event mask will be set explicitly
+                poll_obj_set_events(poll_obj, 0);
             }
             return MP_OBJ_FROM_PTR(t);
         }
@@ -347,7 +605,7 @@ STATIC MP_DEFINE_CONST_OBJ_TYPE(
 // poll()
 STATIC mp_obj_t select_poll(void) {
     mp_obj_poll_t *poll = mp_obj_malloc(mp_obj_poll_t, &mp_type_poll);
-    mp_map_init(&poll->poll_map, 0);
+    poll_set_init(&poll->poll_set, 0);
     poll->iter_cnt = 0;
     poll->ret_tuple = MP_OBJ_NULL;
     return MP_OBJ_FROM_PTR(poll);

extmod/modsocket.c

@@ -631,6 +631,7 @@ STATIC const mp_rom_map_elem_t mp_module_socket_globals_table[] = {
 
     { MP_ROM_QSTR(MP_QSTR_SOL_SOCKET), MP_ROM_INT(MOD_NETWORK_SOL_SOCKET) },
     { MP_ROM_QSTR(MP_QSTR_SO_REUSEADDR), MP_ROM_INT(MOD_NETWORK_SO_REUSEADDR) },
+    { MP_ROM_QSTR(MP_QSTR_SO_BROADCAST), MP_ROM_INT(MOD_NETWORK_SO_BROADCAST) },
     { MP_ROM_QSTR(MP_QSTR_SO_KEEPALIVE), MP_ROM_INT(MOD_NETWORK_SO_KEEPALIVE) },
     { MP_ROM_QSTR(MP_QSTR_SO_SNDTIMEO), MP_ROM_INT(MOD_NETWORK_SO_SNDTIMEO) },
     { MP_ROM_QSTR(MP_QSTR_SO_RCVTIMEO), MP_ROM_INT(MOD_NETWORK_SO_RCVTIMEO) },

extmod/modssl_mbedtls.c

@@ -168,6 +168,12 @@ STATIC mp_obj_t ssl_context_make_new(const mp_obj_type_t *type_in, size_t n_args
     mbedtls_debug_set_threshold(3);
     #endif
 
+    // Whenever the PSA interface is used (if MBEDTLS_PSA_CRYPTO), psa_crypto_init() needs to be called before any TLS related operations.
+    // TLSv1.3 depends on the PSA interface, TLSv1.2 only uses the PSA stack if MBEDTLS_USE_PSA_CRYPTO is defined.
+    #if defined(MBEDTLS_SSL_PROTO_TLS1_3) || defined(MBEDTLS_USE_PSA_CRYPTO)
+    psa_crypto_init();
+    #endif
+
     const byte seed[] = "upy";
     int ret = mbedtls_ctr_drbg_seed(&self->ctr_drbg, mbedtls_entropy_func, &self->entropy, seed, sizeof(seed));
     if (ret != 0) {
@@ -394,6 +400,7 @@ STATIC mp_obj_t ssl_socket_make_new(mp_obj_ssl_context_t *ssl_context, mp_obj_t
     return MP_OBJ_FROM_PTR(o);
 
 cleanup:
+    o->sock = MP_OBJ_NULL;
     mbedtls_ssl_free(&o->ssl);
     mbedtls_raise_error(ret);
 }
@@ -436,6 +443,14 @@ STATIC mp_uint_t socket_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errc
         // renegotiation.
         ret = MP_EWOULDBLOCK;
         o->poll_mask = MP_STREAM_POLL_WR;
+    #if defined(MBEDTLS_SSL_PROTO_TLS1_3)
+    } else if (ret == MBEDTLS_ERR_SSL_RECEIVED_NEW_SESSION_TICKET) {
+        // It appears a new session ticket being issued by the server right after
+        // completed handshake is not uncommon and shouldn't be treated as fatal.
+        // mbedtls itself states "This error code is experimental and may be
+        // changed or removed without notice."
+        ret = MP_EWOULDBLOCK;
+    #endif
     } else {
         o->last_error = ret;
     }
@@ -486,15 +501,20 @@ STATIC mp_uint_t socket_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, i
     mp_uint_t ret = 0;
     uintptr_t saved_arg = 0;
     mp_obj_t sock = self->sock;
-    if (sock == MP_OBJ_NULL || (request != MP_STREAM_CLOSE && self->last_error != 0)) {
-        // Closed or error socket:
-        return MP_STREAM_POLL_NVAL;
-    }
 
     if (request == MP_STREAM_CLOSE) {
+        if (sock == MP_OBJ_NULL) {
+            // Already closed socket, do nothing.
+            return 0;
+        }
         self->sock = MP_OBJ_NULL;
         mbedtls_ssl_free(&self->ssl);
     } else if (request == MP_STREAM_POLL) {
+        if (sock == MP_OBJ_NULL || self->last_error != 0) {
+            // Closed or error socket, return NVAL flag.
+            return MP_STREAM_POLL_NVAL;
+        }
+
         // If the library signaled us that it needs reading or writing, only check that direction,
         // but save what the caller asked because we need to restore it later
         if (self->poll_mask && (arg & MP_STREAM_POLL_RDWR)) {
@@ -514,6 +534,10 @@ STATIC mp_uint_t socket_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, i
                 }
             }
         }
+    } else {
+        // Unsupported ioctl.
+        *errcode = MP_EINVAL;
+        return MP_STREAM_ERROR;
     }
 
     // Pass all requests down to the underlying socket

extmod/vfs_posix_file.c

@@ -153,12 +153,17 @@ STATIC mp_uint_t vfs_posix_file_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_
     switch (request) {
         case MP_STREAM_FLUSH: {
             int ret;
+            // fsync(stdin/stdout/stderr) may fail with EINVAL (or ENOTSUP on macos),
+            // but don't propagate that error out.  Because data is not buffered by
+            // us, and stdin/out/err.flush() should just be a no-op.
+            #ifdef __APPLE__
+            #define VFS_POSIX_STREAM_STDIO_ERR_CATCH (err == EINVAL || err == ENOTSUP)
+            #else
+            #define VFS_POSIX_STREAM_STDIO_ERR_CATCH (err == EINVAL)
+            #endif
             MP_HAL_RETRY_SYSCALL(ret, fsync(o->fd), {
-                if (err == EINVAL
+                if (VFS_POSIX_STREAM_STDIO_ERR_CATCH
                     && (o->fd == STDIN_FILENO || o->fd == STDOUT_FILENO || o->fd == STDERR_FILENO)) {
-                    // fsync(stdin/stdout/stderr) may fail with EINVAL, but don't propagate that
-                    // error out.  Because data is not buffered by us, and stdin/out/err.flush()
-                    // should just be a no-op.
                     return 0;
                 }
                 *errcode = err;
@@ -188,7 +193,7 @@ STATIC mp_uint_t vfs_posix_file_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_
             return 0;
         case MP_STREAM_GET_FILENO:
             return o->fd;
-        #if MICROPY_PY_SELECT
+        #if MICROPY_PY_SELECT && !MICROPY_PY_SELECT_POSIX_OPTIMISATIONS
         case MP_STREAM_POLL: {
             #ifdef _WIN32
             mp_raise_NotImplementedError(MP_ERROR_TEXT("poll on file not available on win32"));
@@ -209,6 +214,15 @@ STATIC mp_uint_t vfs_posix_file_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_
                 if (pfd.revents & POLLOUT) {
                     ret |= MP_STREAM_POLL_WR;
                 }
+                if (pfd.revents & POLLERR) {
+                    ret |= MP_STREAM_POLL_ERR;
+                }
+                if (pfd.revents & POLLHUP) {
+                    ret |= MP_STREAM_POLL_HUP;
+                }
+                if (pfd.revents & POLLNVAL) {
+                    ret |= MP_STREAM_POLL_NVAL;
+                }
             }
             return ret;
             #endif
@@ -260,12 +274,58 @@ STATIC const mp_stream_p_t vfs_posix_textio_stream_p = {
     .is_text = true,
 };
 
+#if MICROPY_PY_SYS_STDIO_BUFFER
+
+const mp_obj_vfs_posix_file_t mp_sys_stdin_buffer_obj = {{&mp_type_vfs_posix_fileio}, STDIN_FILENO};
+const mp_obj_vfs_posix_file_t mp_sys_stdout_buffer_obj = {{&mp_type_vfs_posix_fileio}, STDOUT_FILENO};
+const mp_obj_vfs_posix_file_t mp_sys_stderr_buffer_obj = {{&mp_type_vfs_posix_fileio}, STDERR_FILENO};
+
+// Forward declarations.
+const mp_obj_vfs_posix_file_t mp_sys_stdin_obj;
+const mp_obj_vfs_posix_file_t mp_sys_stdout_obj;
+const mp_obj_vfs_posix_file_t mp_sys_stderr_obj;
+
+STATIC void vfs_posix_textio_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) {
+    if (dest[0] != MP_OBJ_NULL) {
+        // These objects are read-only.
+        return;
+    }
+
+    if (attr == MP_QSTR_buffer) {
+        // Implement the `buffer` attribute only on std{in,out,err} instances.
+        if (MP_OBJ_TO_PTR(self_in) == &mp_sys_stdin_obj) {
+            dest[0] = MP_OBJ_FROM_PTR(&mp_sys_stdin_buffer_obj);
+            return;
+        }
+        if (MP_OBJ_TO_PTR(self_in) == &mp_sys_stdout_obj) {
+            dest[0] = MP_OBJ_FROM_PTR(&mp_sys_stdout_buffer_obj);
+            return;
+        }
+        if (MP_OBJ_TO_PTR(self_in) == &mp_sys_stderr_obj) {
+            dest[0] = MP_OBJ_FROM_PTR(&mp_sys_stderr_buffer_obj);
+            return;
+        }
+    }
+
+    // Any other attribute - forward to locals dict.
+    dest[1] = MP_OBJ_SENTINEL;
+};
+
+#define VFS_POSIX_TEXTIO_TYPE_ATTR attr, vfs_posix_textio_attr,
+
+#else
+
+#define VFS_POSIX_TEXTIO_TYPE_ATTR
+
+#endif // MICROPY_PY_SYS_STDIO_BUFFER
+
 MP_DEFINE_CONST_OBJ_TYPE(
     mp_type_vfs_posix_textio,
     MP_QSTR_TextIOWrapper,
     MP_TYPE_FLAG_ITER_IS_STREAM,
     print, vfs_posix_file_print,
     protocol, &vfs_posix_textio_stream_p,
+    VFS_POSIX_TEXTIO_TYPE_ATTR
     locals_dict, &vfs_posix_rawfile_locals_dict
     );
 

lib/uzlib/header.c

@@ -108,6 +108,10 @@ int uzlib_parse_zlib_gzip_header(uzlib_uncomp_t *d, int *wbits)
         d->checksum_type = UZLIB_CHKSUM_CRC;
         d->checksum = ~0;
 
+        /* gzip does not include the window size in the header, as it is expected that a
+           compressor will use wbits=15 (32kiB).*/
+        *wbits = 15;
+
         return UZLIB_HEADER_GZIP;
     } else {
        /* check checksum */

mpy-cross/mpy_cross/__init__.py

@@ -100,7 +100,7 @@ def compile(src, dest=None, src_path=None, opt=None, march=None, mpy_cross=None,
     if not src:
         raise ValueError("src is required")
     if not os.path.exists(src):
-        raise CrossCompileError("Input .py file not found: {}.".format(src_py))
+        raise CrossCompileError("Input .py file not found: {}.".format(src))
 
     args = []
 

ports/cc3200/Makefile

@@ -7,6 +7,8 @@ endif
 
 # Make 'release' the default build type
 BTYPE ?= release
+# Make the 'application' by default ('bootloader' can be built explicitly)
+BTARGET ?= application
 
 # Port for flashing firmware
 PORT ?= /dev/ttyUSB1

ports/cc3200/application.mk

@@ -198,7 +198,7 @@ WIPY_IP ?= '192.168.1.1'
 WIPY_USER ?= 'micro'
 WIPY_PWD ?= 'python'
 
-all: $(BUILD)/mcuimg.bin
+all: $(BUILD)/firmware.zip
 
 .PHONY: deploy-ota
 
@@ -219,6 +219,10 @@ $(BUILD)/mcuimg.bin: $(BUILD)/application.bin
 	$(ECHO) "Create $@"
 	$(Q)$(SHELL) $(APP_SIGN) $(BUILD)
 
+$(BUILD)/firmware.zip: $(BUILD)/mcuimg.bin
+	$(ECHO) "Create $@"
+	$(Q)$(ZIP) -j $@ $<
+
 MAKE_PINS = boards/make-pins.py
 BOARD_PINS = boards/$(BOARD)/pins.csv
 AF_FILE = boards/cc3200_af.csv

ports/cc3200/boards/WIPY/board.json

@@ -5,13 +5,11 @@
     "docs": "https://docs.pycom.io/datasheets/development/wipy3/",
     "features": [
         "BLE",
-        "Breadboard Friendly",
-        "MicroSD",
+        "External Flash",
         "RGB LED",
-        "SPI Flash",
-        "WiFi"
+        "WiFi",
+        "microSD"
     ],
-    "id": "wipy",
     "images": [
         "wipy.jpg"
     ],

ports/cc3200/mpthreadport.c

@@ -89,6 +89,10 @@ void mp_thread_set_state(mp_state_thread_t *state) {
     vTaskSetThreadLocalStoragePointer(NULL, 0, state);
 }
 
+mp_uint_t mp_thread_get_id(void) {
+    return (mp_uint_t)xTaskGetCurrentTaskHandle();
+}
+
 void mp_thread_start(void) {
     mp_thread_mutex_lock(&thread_mutex, 1);
     for (mp_thread_t *th = thread; th != NULL; th = th->next) {
@@ -111,7 +115,7 @@ STATIC void freertos_entry(void *arg) {
     }
 }
 
-void mp_thread_create(void *(*entry)(void *), void *arg, size_t *stack_size) {
+mp_uint_t mp_thread_create(void *(*entry)(void *), void *arg, size_t *stack_size) {
     // store thread entry function into a global variable so we can access it
     ext_thread_entry = entry;
 
@@ -148,6 +152,9 @@ void mp_thread_create(void *(*entry)(void *), void *arg, size_t *stack_size) {
 
     // adjust stack_size to provide room to recover from hitting the limit
     *stack_size -= 512;
+
+    MP_STATIC_ASSERT(sizeof(mp_uint_t) >= sizeof(TaskHandle_t));
+    return (mp_uint_t)id;
 }
 
 void mp_thread_finish(void) {

ports/cc3200/tools/uniflash.py

@@ -41,7 +41,7 @@ def execute(command):
     if exitCode == 0:
         return cmd_log
     else:
-        raise ProcessException(command, exitCode, output)
+        raise subprocess.CalledProcessError(exitCode, command, output)
 
 
 def main():

ports/esp32/CMakeLists.txt

@@ -4,7 +4,7 @@ cmake_minimum_required(VERSION 3.12)
 
 # Set the board if it's not already set.
 if(NOT MICROPY_BOARD)
-    set(MICROPY_BOARD GENERIC)
+    set(MICROPY_BOARD ESP32_GENERIC)
 endif()
 
 # Set the board directory and check that it exists.

ports/esp32/Makefile

@@ -8,17 +8,25 @@ ifdef BOARD_DIR
 # the path as the board name.
 BOARD ?= $(notdir $(BOARD_DIR:/=))
 else
-# If not given on the command line, then default to GENERIC.
-BOARD ?= GENERIC
+# If not given on the command line, then default to ESP32_GENERIC.
+BOARD ?= ESP32_GENERIC
 BOARD_DIR ?= boards/$(BOARD)
 endif
 
 ifeq ($(wildcard $(BOARD_DIR)/.),)
+ifeq ($(findstring boards/GENERIC,$(BOARD_DIR)),boards/GENERIC)
+$(warning The GENERIC* boards have been renamed to ESP32_GENERIC*)
+endif
 $(error Invalid BOARD specified: $(BOARD_DIR))
 endif
 
-# If the build directory is not given, make it reflect the board name.
+# If the build directory is not given, make it reflect the board name (and
+# optionally the board variant).
+ifneq ($(BOARD_VARIANT),)
+BUILD ?= build-$(BOARD)-$(BOARD_VARIANT)
+else
 BUILD ?= build-$(BOARD)
+endif
 
 # Device serial settings.
 PORT ?= /dev/ttyUSB0
@@ -26,12 +34,6 @@ BAUD ?= 460800
 
 PYTHON ?= python3
 
-# Would be good to use cmake to discover submodules (see how rp2/Makefile does
-# it), but on ESP32 the same trick doesn't work because "idf.py build" fails
-# on berkeley-db dependency before printing out the submodule list.
-# For now just force the submodule dependencies here.
-GIT_SUBMODULES += lib/berkeley-db-1.xx lib/micropython-lib
-
 .PHONY: all clean deploy erase submodules FORCE
 
 CMAKE_ARGS =
@@ -40,20 +42,24 @@ ifdef USER_C_MODULES
 	CMAKE_ARGS += -DUSER_C_MODULES=${USER_C_MODULES}
 endif
 
-IDFPY_FLAGS += -D MICROPY_BOARD=$(BOARD) -D MICROPY_BOARD_DIR=$(abspath $(BOARD_DIR)) -B $(BUILD) $(CMAKE_ARGS)
+IDFPY_FLAGS += -D MICROPY_BOARD=$(BOARD) -D MICROPY_BOARD_DIR=$(abspath $(BOARD_DIR)) $(CMAKE_ARGS)
 
 ifdef FROZEN_MANIFEST
        IDFPY_FLAGS += -D MICROPY_FROZEN_MANIFEST=$(FROZEN_MANIFEST)
 endif
 
+ifdef BOARD_VARIANT
+	IDFPY_FLAGS += -D MICROPY_BOARD_VARIANT=$(BOARD_VARIANT)
+endif
+
 HELP_BUILD_ERROR ?= "See \033[1;31mhttps://github.com/micropython/micropython/wiki/Build-Troubleshooting\033[0m"
 
 define RUN_IDF_PY
-	idf.py $(IDFPY_FLAGS) -p $(PORT) -b $(BAUD) $(1)
+	idf.py $(IDFPY_FLAGS) -B $(BUILD) -p $(PORT) -b $(BAUD) $(1)
 endef
 
 all:
-	idf.py $(IDFPY_FLAGS) build || (echo -e $(HELP_BUILD_ERROR); false)
+	idf.py $(IDFPY_FLAGS) -B $(BUILD) build || (echo -e $(HELP_BUILD_ERROR); false)
 	@$(PYTHON) makeimg.py \
 		$(BUILD)/sdkconfig \
 		$(BUILD)/bootloader/bootloader.bin \
@@ -85,5 +91,12 @@ size-components:
 size-files:
 	$(call RUN_IDF_PY,size-files)
 
+# Running the build with ECHO_SUBMODULES set will trigger py/mkrules.cmake to
+# print out the value of the GIT_SUBMODULES variable, prefixed with
+# "GIT_SUBMODULES", and then abort. This extracts out that line from the idf.py
+# output and passes the list of submodules to py/mkrules.mk which does the
+# `git submodule init` on each.
 submodules:
-	$(MAKE) -f ../../py/mkrules.mk GIT_SUBMODULES="$(GIT_SUBMODULES)" submodules
+	@GIT_SUBMODULES=$$(idf.py $(IDFPY_FLAGS) -B $(BUILD)/submodules -D ECHO_SUBMODULES=1 build 2>&1 | \
+	                  grep '^GIT_SUBMODULES=' | cut -d= -f2); \
+	$(MAKE) -f ../../py/mkrules.mk GIT_SUBMODULES="$${GIT_SUBMODULES}" submodules

ports/esp32/README.md

@@ -93,7 +93,7 @@ $ make submodules
 $ make
 ```
 
-This will produce a combined `firmware.bin` image in the `build-GENERIC/`
+This will produce a combined `firmware.bin` image in the `build-ESP32_GENERIC/`
 subdirectory (this firmware image is made up of: bootloader.bin, partitions.bin
 and micropython.bin).
 
@@ -123,12 +123,12 @@ To flash the MicroPython firmware to your ESP32 use:
 $ make deploy
 ```
 
-The default ESP32 board build by the above commands is the `GENERIC` one, which
-should work on most ESP32 modules.  You can specify a different board by passing
-`BOARD=<board>` to the make commands, for example:
+The default ESP32 board build by the above commands is the `ESP32_GENERIC`
+one, which should work on most ESP32 modules.  You can specify a different
+board by passing `BOARD=<board>` to the make commands, for example:
 
 ```bash
-$ make BOARD=GENERIC_SPIRAM
+$ make BOARD=ESP32_GENERIC_S3
 ```
 
 Note: the above "make" commands are thin wrappers for the underlying `idf.py`
@@ -137,10 +137,25 @@ for example:
 
 ```bash
 $ idf.py build
-$ idf.py -D MICROPY_BOARD=GENERIC_SPIRAM build
+$ idf.py -D MICROPY_BOARD=ESP32_GENERIC build
 $ idf.py flash
 ```
 
+Some boards also support "variants", which are allow for small variations of
+an otherwise similar board. For example different flash sizes or features. For
+example to build the `OTA` variant of `ESP32_GENERIC`.
+
+```bash
+$ make BOARD=ESP32_GENERIC BOARD_VARIANT=OTA
+```
+
+or to enable octal-SPIRAM support for the `ESP32_GENERIC_S3` board:
+
+```bash
+$ make BOARD=ESP32_GENERIC BOARD_VARIANT=SPIRAM_OCT
+```
+
+
 Getting a Python prompt on the device
 -------------------------------------
 
@@ -202,10 +217,10 @@ antenna = machine.Pin(16, machine.Pin.OUT, value=0)
 Defining a custom ESP32 board
 -----------------------------
 
-The default ESP-IDF configuration settings are provided by the `GENERIC`
-board definition in the directory `boards/GENERIC`. For a custom configuration
+The default ESP-IDF configuration settings are provided by the `ESP32_GENERIC`
+board definition in the directory `boards/ESP32_GENERIC`. For a custom configuration
 you can define your own board directory.  Start a new board configuration by
-copying an existing one (like `GENERIC`) and modifying it to suit your board.
+copying an existing one (like `ESP32_GENERIC`) and modifying it to suit your board.
 
 MicroPython specific configuration values are defined in the board-specific
 `mpconfigboard.h` file, which is included by `mpconfigport.h`.  Additional

ports/esp32/boards/ARDUINO_NANO_ESP32/board.json

@@ -5,9 +5,10 @@
     "docs": "",
     "features": [
         "BLE",
-        "WiFi",
+        "External Flash",
+        "RGB LED",
         "USB-C",
-        "RGB LED"
+        "WiFi"
     ],
     "images": [
         "ABX00092_01.iso_1000x750.jpg"

ports/esp32/boards/ARDUINO_NANO_ESP32/deploy.md

@@ -4,5 +4,10 @@ This board can programmed via DFU bootloader, using e.g. [dfu-util](http://dfu-u
 To enter the DFU bootloader, double tap the reset (blue) button, or you can use `machine.bootloader()` from the MicroPython REPL.
 
 ```bash
-dfu-util -d 0x2341:0x0070 -R -D build-ARDUINO_NANO_ESP32/micropython.bin
+dfu-util -d 0x2341:0x0070 -R -D build-ARDUINO_NANO_ESP32/micropython.app-bin
 ```
+
+Please note that the DFU bootloader comes factory flashed. Should you for any reason erase the
+entire flash, the DFU bootloader will have to be re-installed. Please follow the instructions
+[here](https://support.arduino.cc/hc/en-us/articles/9810414060188-Reset-the-Arduino-bootloader-on-the-Nano-ESP32)
+to do so.

ports/esp32/boards/ESP32_GENERIC/board.json

@@ -5,18 +5,22 @@
     "docs": "",
     "features": [
         "BLE",
+        "External Flash",
         "WiFi"
     ],
-    "id": "esp32",
     "images": [
         "esp32_devkitc.jpg"
     ],
     "mcu": "esp32",
-    "product": "ESP32",
+    "product": "ESP32 / WROOM",
     "thumbnail": "",
     "url": "https://www.espressif.com/en/products/modules",
     "variants": {
-        "idf3": "Compiled with IDF 3.x"
+        "IDF3": "Compiled with IDF 3.x",
+        "D2WD": "ESP32 D2WD",
+        "SPIRAM": "Support for SPIRAM / WROVER",
+        "UNICORE": "ESP32 Unicore",
+        "OTA": "Support for OTA"
     },
     "vendor": "Espressif"
 }

ports/esp32/boards/ESP32_GENERIC/board.md

@@ -0,0 +1,9 @@
+The following files are firmware that should work on most ESP32-based boards
+with 4MiB of flash, including WROOM WROVER, SOLO, PICO, and MINI modules.
+
+If your board is based on a WROVER module, or otherwise has SPIRAM (also known
+as PSRAM), then use the "spiram" variant.
+
+The "d2wd" variant is for ESP32-D2WD chips (with 2MiB flash), and "unicore" is
+for single-core ESP32 chips (e.g. the "SOLO" modules). The "ota" variant sets
+up the partition table to allow for Over-the-Air updates.

ports/esp32/boards/ESP32_GENERIC/mpconfigboard.cmake

@@ -0,0 +1,48 @@
+set(SDKCONFIG_DEFAULTS
+    boards/sdkconfig.base
+    boards/sdkconfig.ble
+)
+
+if(MICROPY_BOARD_VARIANT STREQUAL "D2WD")
+    set(SDKCONFIG_DEFAULTS
+        ${SDKCONFIG_DEFAULTS}
+        boards/ESP32_GENERIC/sdkconfig.d2wd
+    )
+
+    list(APPEND MICROPY_DEF_BOARD
+        MICROPY_HW_MCU_NAME="ESP32-D2WD"
+    )
+endif()
+
+if(MICROPY_BOARD_VARIANT STREQUAL "OTA")
+    set(SDKCONFIG_DEFAULTS
+        ${SDKCONFIG_DEFAULTS}
+        boards/ESP32_GENERIC/sdkconfig.ota
+    )
+
+    list(APPEND MICROPY_DEF_BOARD
+        MICROPY_HW_BOARD_NAME="Generic ESP32 module with OTA"
+    )
+endif()
+
+if(MICROPY_BOARD_VARIANT STREQUAL "SPIRAM")
+    set(SDKCONFIG_DEFAULTS
+        ${SDKCONFIG_DEFAULTS}
+        boards/sdkconfig.spiram
+    )
+
+    list(APPEND MICROPY_DEF_BOARD
+        MICROPY_HW_BOARD_NAME="Generic ESP32 module with SPIRAM"
+    )
+endif()
+
+if(MICROPY_BOARD_VARIANT STREQUAL "UNICORE")
+    set(SDKCONFIG_DEFAULTS
+        ${SDKCONFIG_DEFAULTS}
+        boards/ESP32_GENERIC/sdkconfig.unicore
+    )
+
+    list(APPEND MICROPY_DEF_BOARD
+        MICROPY_HW_MCU_NAME="ESP32-UNICORE"
+    )
+endif()

ports/esp32/boards/ESP32_GENERIC/mpconfigboard.h

@@ -0,0 +1,10 @@
+// Both of these can be set by mpconfigboard.cmake if a BOARD_VARIANT is
+// specified.
+
+#ifndef MICROPY_HW_BOARD_NAME
+#define MICROPY_HW_BOARD_NAME "Generic ESP32 module"
+#endif
+
+#ifndef MICROPY_HW_MCU_NAME
+#define MICROPY_HW_MCU_NAME "ESP32"
+#endif

ports/esp32/boards/ESP32_GENERIC/sdkconfig.ota

@@ -1,6 +1,6 @@
 CONFIG_BOOTLOADER_APP_ROLLBACK_ENABLE=y
 CONFIG_PARTITION_TABLE_CUSTOM=y
-CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions-ota.csv"
+CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions-4MiB-ota.csv"
 
 # Reduce firmware size to fit in the OTA partition.
 CONFIG_COMPILER_OPTIMIZATION_CHECKS_SILENT=y

ports/esp32/boards/ESP32_GENERIC_C3/board.json

@@ -5,9 +5,9 @@
     "docs": "",
     "features": [
         "BLE",
+        "External Flash",
         "WiFi"
     ],
-    "id": "esp32c3",
     "images": [
         "esp32c3_devkitmini.jpg"
     ],

ports/esp32/boards/ESP32_GENERIC_C3/board.md

@@ -0,0 +1,7 @@
+The following files are firmware images that should work on most
+ESP32-C3-based boards with 4MiB of flash, including WROOM and MINI modules,
+that use the revision 3 silicon (or newer).
+
+USB serial/JTAG support is enabled on pin 18 and 19. Note that this
+is not a full USB stack, the C3 just provides a CDC/ACM class serial
+and JTAG interface.

ports/esp32/boards/ESP32_GENERIC_C3/mpconfigboard.cmake

@@ -3,4 +3,5 @@ set(IDF_TARGET esp32c3)
 set(SDKCONFIG_DEFAULTS
     boards/sdkconfig.base
     boards/sdkconfig.ble
+    boards/ESP32_GENERIC_C3/sdkconfig.c3usb
 )

ports/esp32/boards/ESP32_GENERIC_C3/mpconfigboard.h

@@ -6,3 +6,6 @@
 #define MICROPY_HW_ENABLE_SDCARD            (0)
 #define MICROPY_PY_MACHINE_DAC              (0)
 #define MICROPY_PY_MACHINE_I2S              (0)
+
+// Enable UART REPL for modules that have an external USB-UART and don't use native USB.
+#define MICROPY_HW_ENABLE_UART_REPL         (1)

ports/esp32/boards/ESP32_GENERIC_S2/board.json

@@ -5,6 +5,8 @@
     "docs": "",
     "features": [
         "BLE",
+        "External Flash",
+        "External RAM",
         "WiFi"
     ],
     "images": [

ports/esp32/boards/ESP32_GENERIC_S2/board.md

@@ -0,0 +1,6 @@
+The following files are firmware that should work on most ESP32-S2-based
+boards with 4MiB of flash, including WROOM, WROVER, and MINI modules.
+
+This firmware supports configurations with and without SPIRAM (also known as
+PSRAM) and will auto-detect a connected SPIRAM chip at startup and allocate
+the MicroPython heap accordingly.

ports/esp32/boards/ESP32_GENERIC_S2/mpconfigboard.cmake

@@ -3,4 +3,5 @@ set(IDF_TARGET esp32s2)
 set(SDKCONFIG_DEFAULTS
     boards/sdkconfig.base
     boards/sdkconfig.usb
+    boards/sdkconfig.spiram_sx
 )

ports/esp32/boards/ESP32_GENERIC_S2/mpconfigboard.h

@@ -0,0 +1,8 @@
+#define MICROPY_HW_BOARD_NAME "Generic ESP32S2 module"
+#define MICROPY_HW_MCU_NAME "ESP32S2"
+
+#define MICROPY_PY_BLUETOOTH (0)
+#define MICROPY_HW_ENABLE_SDCARD (0)
+
+// Enable UART REPL for modules that have an external USB-UART and don't use native USB.
+#define MICROPY_HW_ENABLE_UART_REPL         (1)

ports/esp32/boards/ESP32_GENERIC_S3/board.json

@@ -5,6 +5,8 @@
     "docs": "",
     "features": [
         "BLE",
+        "External Flash",
+        "External RAM",
         "WiFi"
     ],
     "images": [
@@ -14,5 +16,8 @@
     "product": "ESP32-S3",
     "thumbnail": "",
     "url": "https://www.espressif.com/en/products/modules",
-    "vendor": "Espressif"
+    "vendor": "Espressif",
+    "variants": {
+        "SPIRAM_OCT": "Support for Octal-SPIRAM"
+    }
 }

ports/esp32/boards/ESP32_GENERIC_S3/board.md

@@ -0,0 +1,7 @@
+The following files are firmware that should work on most ESP32-S3-based
+boards with 8MiB of flash, including WROOM and MINI modules.
+
+This firmware supports configurations with and without SPIRAM (also known as
+PSRAM) and will auto-detect a connected SPIRAM chip at startup and allocate
+the MicroPython heap accordingly. However if your board has Octal SPIRAM, then
+use the "spiram-oct" variant.

ports/esp32/boards/ESP32_GENERIC_S3/mpconfigboard.cmake

@@ -0,0 +1,21 @@
+set(IDF_TARGET esp32s3)
+
+set(SDKCONFIG_DEFAULTS
+    boards/sdkconfig.base
+    boards/sdkconfig.usb
+    boards/sdkconfig.ble
+    boards/sdkconfig.spiram_sx
+    boards/ESP32_GENERIC_S3/sdkconfig.board
+)
+
+if(MICROPY_BOARD_VARIANT STREQUAL "SPIRAM_OCT")
+    set(SDKCONFIG_DEFAULTS
+        ${SDKCONFIG_DEFAULTS}
+        boards/sdkconfig.240mhz
+        boards/sdkconfig.spiram_oct
+    )
+
+    list(APPEND MICROPY_DEF_BOARD
+        MICROPY_HW_BOARD_NAME="Generic ESP32S3 module with Octal-SPIRAM"
+    )
+endif()

ports/esp32/boards/ESP32_GENERIC_S3/mpconfigboard.h

@@ -1,4 +1,7 @@
-#define MICROPY_HW_BOARD_NAME               "ESP32S3 module (spiram)"
+#ifndef MICROPY_HW_BOARD_NAME
+// Can be set by mpconfigboard.cmake.
+#define MICROPY_HW_BOARD_NAME               "Generic ESP32S3 module"
+#endif
 #define MICROPY_HW_MCU_NAME                 "ESP32S3"
 
 #define MICROPY_PY_MACHINE_DAC              (0)