From 05dcb8be9957d0ed6c9694629be186a40c1a3fd9 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Wed, 2 Aug 2023 16:51:07 +1000 Subject: [PATCH] esp32: Enable automatic Python heap growth. Via MICROPY_GC_SPLIT_HEAP_AUTO feature flag added in previous commit. Tested on ESP32 GENERIC_SPIRAM and GENERIC_S3 configurations, with some worst-case allocation patterns and the standard test suite. This work was funded through GitHub Sponsors. Signed-off-by: Angus Gratton --- docs/library/esp32.rst | 11 +++++++++-- ports/esp32/gccollect.c | 10 ++++++++++ ports/esp32/main.c | 14 ++++++-------- ports/esp32/mpconfigport.h | 3 +++ 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/docs/library/esp32.rst b/docs/library/esp32.rst index efdd6c1be2..856d9aef8d 100644 --- a/docs/library/esp32.rst +++ b/docs/library/esp32.rst @@ -51,13 +51,20 @@ Functions buffers and other data. This data is useful to get a sense of how much memory is available to ESP-IDF and the networking stack in particular. It may shed some light on situations where ESP-IDF operations fail due to allocation failures. - The information returned is *not* useful to troubleshoot Python allocation failures, - use `micropython.mem_info()` instead. The capabilities parameter corresponds to ESP-IDF's ``MALLOC_CAP_XXX`` values but the two most useful ones are predefined as `esp32.HEAP_DATA` for data heap regions and `esp32.HEAP_EXEC` for executable regions as used by the native code emitter. + Free IDF heap memory in the `esp32.HEAP_DATA` region is available to be + automatically added to the MicroPython heap to prevent a MicroPython + allocation from failing. However, the information returned here is otherwise + *not* useful to troubleshoot Python allocation failures, use + `micropython.mem_info()` instead. The "max new split" value in + `micropython.mem_info()` output corresponds to the largest free block of + ESP-IDF heap that could be automatically added on demand to the MicroPython + heap. + The return value is a list of 4-tuples, where each 4-tuple corresponds to one heap and contains: the total bytes, the free bytes, the largest free block, and the minimum free seen over time. diff --git a/ports/esp32/gccollect.c b/ports/esp32/gccollect.c index 6fa287de28..e16e8028ad 100644 --- a/ports/esp32/gccollect.c +++ b/ports/esp32/gccollect.c @@ -80,3 +80,13 @@ void gc_collect(void) { } #endif + +#if MICROPY_GC_SPLIT_HEAP_AUTO + +// The largest new region that is available to become Python heap is the largest +// free block in the ESP-IDF system heap. +size_t gc_get_max_new_split(void) { + return heap_caps_get_largest_free_block(MALLOC_CAP_DEFAULT); +} + +#endif diff --git a/ports/esp32/main.c b/ports/esp32/main.c index 3a172e6f8d..a6346b027a 100644 --- a/ports/esp32/main.c +++ b/ports/esp32/main.c @@ -75,6 +75,10 @@ #define MP_TASK_STACK_LIMIT_MARGIN (1024) #endif +// Initial Python heap size. This starts small but adds new heap areas on +// demand due to settings MICROPY_GC_SPLIT_HEAP & MICROPY_GC_SPLIT_HEAP_AUTO +#define MP_TASK_HEAP_SIZE (64 * 1024) + int vprintf_null(const char *format, va_list ap) { // do nothing: this is used as a log target during raw repl mode return 0; @@ -100,19 +104,13 @@ void mp_task(void *pvParameter) { ESP_LOGE("esp_init", "can't create event loop: 0x%x\n", err); } - // Allocate the uPy heap using malloc and get the largest available region, - // limiting to 1/2 total available memory to leave memory for the OS. - // When SPIRAM is enabled, this will allocate from SPIRAM. - uint32_t caps = MALLOC_CAP_8BIT; - size_t heap_total = heap_caps_get_total_size(caps); - size_t mp_task_heap_size = MIN(heap_caps_get_largest_free_block(caps), heap_total / 2); - void *mp_task_heap = heap_caps_malloc(mp_task_heap_size, caps); + void *mp_task_heap = MP_PLAT_ALLOC_HEAP(MP_TASK_HEAP_SIZE); soft_reset: // initialise the stack pointer for the main thread mp_stack_set_top((void *)sp); mp_stack_set_limit(MP_TASK_STACK_SIZE - MP_TASK_STACK_LIMIT_MARGIN); - gc_init(mp_task_heap, mp_task_heap + mp_task_heap_size); + gc_init(mp_task_heap, mp_task_heap + MP_TASK_HEAP_SIZE); mp_init(); mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_lib)); readline_init0(); diff --git a/ports/esp32/mpconfigport.h b/ports/esp32/mpconfigport.h index aa13eaf2fe..55035b5f86 100644 --- a/ports/esp32/mpconfigport.h +++ b/ports/esp32/mpconfigport.h @@ -68,6 +68,9 @@ #define MICROPY_PY_THREAD_GIL (1) #define MICROPY_PY_THREAD_GIL_VM_DIVISOR (32) +#define MICROPY_GC_SPLIT_HEAP (1) +#define MICROPY_GC_SPLIT_HEAP_AUTO (1) + // extended modules #ifndef MICROPY_ESPNOW #define MICROPY_ESPNOW (1)