Merge pull request #878 from pimoroni/ci/tooling

CI: Move some workflow steps into ci/micropython.sh

.github/workflows/micropython.yml

@@ -7,63 +7,11 @@ on:
     types: [created]
 
 env:
-  MICROPYTHON_VERSION: v1.21.0
+  MICROPYTHON_VERSION: v1.22.1
-  WORKFLOW_VERSION: v1
 
 jobs:
-  deps:
-    runs-on: ubuntu-20.04
-    name: Dependencies
-    steps:
-    - name: Workspace Cache
-      id: cache
-      uses: actions/cache@v3
-      with:
-        path: ${{runner.workspace}}
-        key: workspace-micropython-${{env.MICROPYTHON_VERSION}}-${{env.WORKFLOW_VERSION}}
-        restore-keys: |
-          workspace-micropython-${{env.MICROPYTHON_VERSION}}-${{env.WORKFLOW_VERSION}}
-
-    # Check out MicroPython
-    - name: Checkout MicroPython
-      if: steps.cache.outputs.cache-hit != 'true'
-      uses: actions/checkout@v3
-      with:
-        repository: micropython/micropython
-        ref: ${{env.MICROPYTHON_VERSION}}
-        submodules: false  # MicroPython submodules are hideously broken
-        path: micropython
-
-    # Check out MicroPython Libs
-    - name: Checkout MicroPython Libs
-      if: steps.cache.outputs.cache-hit != 'true'
-      uses: actions/checkout@v3
-      with:
-        repository: micropython/micropython-lib
-        path: micropython-lib
-
-    - name: Fetch Pico submodules
-      if: steps.cache.outputs.cache-hit != 'true'
-      shell: bash
-      working-directory: micropython/ports/rp2
-      run: |
-        git submodule update --init ../../lib/pico-sdk
-        git submodule update --init ../../lib/cyw43-driver
-        git submodule update --init ../../lib/lwip
-        git submodule update --init ../../lib/mbedtls
-        git submodule update --init ../../lib/micropython-lib
-        git submodule update --init ../../lib/tinyusb
-        git submodule update --init ../../lib/btstack
-
-    - name: Build mpy-cross
-      if: steps.cache.outputs.cache-hit != 'true'
-      shell: bash
-      working-directory: micropython/mpy-cross
-      run: make
-
   build:
-    needs: deps
+    name: ${{ matrix.name }} (${{ matrix.board }})
-    name: ${{matrix.name}} (${{matrix.board}})
     runs-on: ubuntu-20.04
     strategy:
       matrix:
@@ -82,7 +30,6 @@ jobs:
           board: PIMORONI_TUFTY2040
         - name: enviro
           board: PICO_W_ENVIRO
-          patch: true
         - name: galactic_unicorn
           board: RPI_PICO_W
         - name: cosmic_unicorn
@@ -91,93 +38,91 @@ jobs:
           board: RPI_PICO_W
         - name: inky_frame
           board: PICO_W_INKY
-          patch: true
 
     env:
       # MicroPython version will be contained in github.event.release.tag_name for releases
-      RELEASE_FILE: pimoroni-${{matrix.name}}-${{github.event.release.tag_name || github.sha}}-micropython
+      RELEASE_FILE: pimoroni-${{ matrix.name }}-${{ github.event.release.tag_name || github.sha }}-micropython
-      MICROPY_BOARD_DIR: "$GITHUB_WORKSPACE/pimoroni-pico-${{ github.sha }}/micropython/board/${{ matrix.BOARD }}"
+      PIMORONI_PICO_DIR: "${{ github.workspace }}/pimoroni-pico-${{ github.sha }}"
-      USER_C_MODULES: "$GITHUB_WORKSPACE/pimoroni-pico-${{ github.sha }}/micropython/modules/micropython-${{matrix.name}}.cmake"
+      MICROPY_BOARD_DIR: "${{ github.workspace }}/pimoroni-pico-${{ github.sha }}/micropython/board/${{ matrix.BOARD }}"
+      USER_C_MODULES: "${{ github.workspace }}/pimoroni-pico-${{ github.sha }}/micropython/modules/micropython-${{ matrix.name }}.cmake"
+      TAG_OR_SHA: ${{ github.event.release.tag_name || github.sha }}
+      MICROPY_BOARD: ${{ matrix.board }}
+      BOARD_NAME: ${{ matrix.name }}
+      BUILD_TOOLS: pimoroni-pico-${{ github.sha }}/ci/micropython.sh
 
     steps:
     - name: Compiler Cache
       uses: actions/cache@v3
       with:
         path: /home/runner/.ccache
-        key: ccache-micropython-${{matrix.name}}-${{github.ref}}-${{github.sha}}
+        key: ccache-micropython-${{ matrix.name }}-${{ github.ref }}-${{ github.sha }}
         restore-keys: |
-          ccache-micropython-${{matrix.name}}-${{github.ref}}
+          ccache-micropython-${{ matrix.name }}-${{ github.ref }}
-          ccache-micropython-${{matrix.name}}-
+          ccache-micropython-${{ matrix.name }}-
-  
-    - name: Workspace Cache
-      uses: actions/cache@v3
-      with:
-        path: ${{runner.workspace}}
-        key: workspace-micropython-${{env.MICROPYTHON_VERSION}}-${{env.WORKFLOW_VERSION}}
-        restore-keys: |
-          workspace-micropython-${{env.MICROPYTHON_VERSION}}-${{env.WORKFLOW_VERSION}}
 
-    - name: Install Compiler & CCache
+    - uses: actions/checkout@v4
-      if: runner.os == 'Linux'
-      run: |
-        sudo apt update && sudo apt install ccache gcc-arm-none-eabi
-  
-    - uses: actions/checkout@v3
       with:
         submodules: true
         path: pimoroni-pico-${{ github.sha }}
 
-    - name: Set MicroPython Version Env Vars
+    - name: Install Arm GNU Toolchain (arm-none-eabi-gcc)
-      shell: bash
+      uses: carlosperate/arm-none-eabi-gcc-action@v1
+      with:
+        release: '9-2020-q2'
+
+    - name: Install CCache
       run: |
-        echo "MICROPY_GIT_TAG=$MICROPYTHON_VERSION, ${{matrix.name}} ${{github.event.release.tag_name || github.sha}}" >> $GITHUB_ENV
+        source $BUILD_TOOLS
-        echo "MICROPY_GIT_HASH=$MICROPYTHON_VERSION-${{github.event.release.tag_name || github.sha}}" >> $GITHUB_ENV
+        apt_install_build_deps
+      
+    - name: Checkout MicroPython & Submodules
+      run: |
+        source $BUILD_TOOLS
+        micropython_clone
+
+    - name: Build MPY Cross
+      run: |
+        source $BUILD_TOOLS
+        micropython_build_mpy_cross
 
     - name: "HACK: CMakeLists.txt Disable C++ Exceptions Patch"
       shell: bash
-      working-directory: micropython
+      run: |
-      run: git apply $GITHUB_WORKSPACE/pimoroni-pico-${{ github.sha }}/micropython/micropython_nano_specs.patch
+        source $BUILD_TOOLS
+        hack_patch_micropython_disable_exceptions
 
     - name: "HACK: Pico SDK Patch"
-      if: matrix.patch == true
       shell: bash
-      working-directory: micropython
       run: |
-        $GITHUB_WORKSPACE/pimoroni-pico-${{ github.sha }}/micropython/board/pico-sdk-patch.sh ${{matrix.board}}
+        source $BUILD_TOOLS
+        hack_patch_pico_sdk
 
     - name: Configure MicroPython
       shell: bash
-      working-directory: micropython/ports/rp2
       run: |
-        cmake -S . -B build-${{matrix.name}} -DPICO_BUILD_DOCS=0 -DUSER_C_MODULES=${{env.USER_C_MODULES}} -DMICROPY_BOARD_DIR=${{env.MICROPY_BOARD_DIR}} -DMICROPY_BOARD=${{matrix.board}} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+        source $BUILD_TOOLS
+        micropython_version
+        cmake_configure
 
-    - name: Build MicroPython  # Multiple simultaneous jobs trigger https://github.com/pimoroni/pimoroni-pico/issues/761
+    - name: Build MicroPython
       shell: bash
-      working-directory: micropython/ports/rp2
       run: |
-        ccache --zero-stats || true
+        source $BUILD_TOOLS
-        cmake --build build-${{matrix.name}} -j 1
+        cmake_build
-        ccache --show-stats || true
-
-    - name: Rename .uf2 for artifact
-      shell: bash
-      working-directory: micropython/ports/rp2/build-${{matrix.name}}
-      run: |
-        cp firmware.uf2 $RELEASE_FILE.uf2
 
     - name: Store .uf2 as artifact
       uses: actions/upload-artifact@v3
       with:
-        name: ${{env.RELEASE_FILE}}.uf2
+        name: ${{ env.RELEASE_FILE }}.uf2
-        path: micropython/ports/rp2/build-${{matrix.name}}/${{env.RELEASE_FILE}}.uf2
+        path: build-${{ matrix.name }}/${{ env.RELEASE_FILE }}.uf2
 
     - name: Upload .uf2
       if: github.event_name == 'release'
       uses: actions/upload-release-asset@v1
       env:
-        GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
-        asset_path: micropython/ports/rp2/build-${{matrix.name}}/firmware.uf2
+        asset_path: build-${{ matrix.name }}/firmware.uf2
-        upload_url: ${{github.event.release.upload_url}}
+        upload_url: ${{ github.event.release.upload_url }}
-        asset_name: ${{env.RELEASE_FILE}}.uf2
+        asset_name: ${{ env.RELEASE_FILE }}.uf2
         asset_content_type: application/octet-stream

ci/micropython.sh

@@ -0,0 +1,75 @@
+export TERM=${TERM:="xterm-256color"}
+
+function log_success {
+	echo -e "$(tput setaf 2)$1$(tput sgr0)"
+}
+
+function log_inform {
+	echo -e "$(tput setaf 6)$1$(tput sgr0)"
+}
+
+function log_warning {
+	echo -e "$(tput setaf 1)$1$(tput sgr0)"
+}
+
+function micropython_clone {
+    log_inform "Using MicroPython $MICROPYTHON_VERSION"
+    git clone https://github.com/micropython/micropython --depth=1 --branch=$MICROPYTHON_VERSION
+    cd micropython
+    git submodule update --init lib/pico-sdk
+    git submodule update --init lib/cyw43-driver
+    git submodule update --init lib/lwip
+    git submodule update --init lib/mbedtls
+    git submodule update --init lib/micropython-lib
+    git submodule update --init lib/tinyusb
+    git submodule update --init lib/btstack
+    cd ../
+}
+
+function micropython_build_mpy_cross {
+    cd micropython/mpy-cross
+    ccache --zero-stats || true
+    CROSS_COMPILE="ccache " make
+    ccache --show-stats || true
+    cd ../../
+}
+
+function apt_install_build_deps {
+    sudo apt update && sudo apt install ccache
+}
+
+function micropython_version {
+    echo "MICROPY_GIT_TAG=$MICROPYTHON_VERSION, $BOARD_NAME $TAG_OR_SHA" >> $GITHUB_ENV
+    echo "MICROPY_GIT_HASH=$MICROPYTHON_VERSION-$TAG_OR_SHA" >> $GITHUB_ENV
+}
+
+function hack_patch_micropython_disable_exceptions {
+    cd micropython
+    git apply $PIMORONI_PICO_DIR/micropython/micropython_nano_specs.patch
+    cd ../
+}
+
+function hack_patch_pico_sdk {
+    # pico-sdk-patch.sh will apply the patch if it exists
+    cd micropython
+    $PIMORONI_PICO_DIR/micropython/board/pico-sdk-patch.sh $MICROPY_BOARD
+    cd ../
+}
+
+function cmake_configure {
+    cmake -S micropython/ports/rp2 -B build-$BOARD_NAME \
+    -DPICO_BUILD_DOCS=0 \
+    -DUSER_C_MODULES=$USER_C_MODULES \
+    -DMICROPY_BOARD_DIR=$MICROPY_BOARD_DIR \
+    -DMICROPY_BOARD=$MICROPY_BOARD \
+    -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+    -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+}
+
+function cmake_build {
+    ccache --zero-stats || true
+    cmake --build build-$BOARD_NAME -j 2
+    ccache --show-stats || true
+    cd build-$BOARD_NAME
+    cp firmware.uf2 $RELEASE_FILE.uf2
+}

micropython/board/pico-sdk-patch.sh

@@ -7,4 +7,6 @@ if [[ -f "$FIXUP_DIR/$BOARD/pico_sdk.patch" ]]; then
     echo "Applying pico_sdk.patch"
     cd $MPY_DIR/lib/pico-sdk
     git apply "$FIXUP_DIR/$BOARD/pico_sdk.patch"
+else
+    echo "Skipping patch. $FIXUP_DIR/$BOARD/pico_sdk.patch not found!"
 fi

micropython/micropython_nano_specs.patch

@@ -1,11 +1,11 @@
 diff --git a/ports/rp2/CMakeLists.txt b/ports/rp2/CMakeLists.txt
-index 094031c6852a..5f268414c08f 100644
+index 281b0c3bc..7e04bb549 100644
 --- a/ports/rp2/CMakeLists.txt
 +++ b/ports/rp2/CMakeLists.txt
-@@ -374,6 +374,15 @@ target_compile_options(${MICROPY_TARGET} PRIVATE
+@@ -453,6 +453,16 @@ target_link_options(${MICROPY_TARGET} PRIVATE
- target_link_options(${MICROPY_TARGET} PRIVATE
+     -Wl,--wrap=dcd_event_handler
-     -Wl,--defsym=__micropy_c_heap_size__=${MICROPY_C_HEAP_SIZE}
  )
+ 
 +# Do not include stack unwinding & exception handling for C++ user modules
 +target_compile_definitions(usermod INTERFACE PICO_CXX_ENABLE_EXCEPTIONS=0)
 +target_compile_options(usermod INTERFACE $<$<COMPILE_LANGUAGE:CXX>:
@@ -15,6 +15,7 @@ index 094031c6852a..5f268414c08f 100644
 +    -fno-use-cxa-atexit
 +>)
 +target_link_options(usermod INTERFACE -specs=nano.specs)
- 
++
  set_source_files_properties(
      ${PICO_SDK_PATH}/src/rp2_common/pico_double/double_math.c
+     ${PICO_SDK_PATH}/src/rp2_common/pico_float/float_math.c

micropython/modules/badger2040/badger2040.cpp

@@ -122,8 +122,8 @@ mp_obj_t Badger2040_update(mp_obj_t self_in) {
     _Badger2040_obj_t *self = MP_OBJ_TO_PTR2(self_in, _Badger2040_obj_t);
 
     while(self->badger2040->is_busy()) {
-#ifdef MICROPY_EVENT_POLL_HOOK
+#ifdef mp_event_handle_nowait
-MICROPY_EVENT_POLL_HOOK
+mp_event_handle_nowait();
 #endif
     }
 
@@ -133,8 +133,8 @@ MICROPY_EVENT_POLL_HOOK
     // Ensure blocking for the minimum amount of time
     // in cases where "is_busy" is unreliable.
     while(self->badger2040->is_busy() || absolute_time_diff_us(get_absolute_time(), t_end) > 0) {
-#ifdef MICROPY_EVENT_POLL_HOOK
+#ifdef mp_event_handle_nowait
-MICROPY_EVENT_POLL_HOOK
+mp_event_handle_nowait();
 #endif
     }
 
@@ -166,8 +166,8 @@ mp_obj_t Badger2040_partial_update(size_t n_args, const mp_obj_t *pos_args, mp_m
 
 
     while(self->badger2040->is_busy()) {
-#ifdef MICROPY_EVENT_POLL_HOOK
+#ifdef mp_event_handle_nowait
-MICROPY_EVENT_POLL_HOOK
+mp_event_handle_nowait();
 #endif
     }
 
@@ -177,8 +177,8 @@ MICROPY_EVENT_POLL_HOOK
     // Ensure blocking for the minimum amount of time
     // in cases where "is_busy" is unreliable.
     while(self->badger2040->is_busy() || absolute_time_diff_us(get_absolute_time(), t_end) > 0) {
-#ifdef MICROPY_EVENT_POLL_HOOK
+#ifdef mp_event_handle_nowait
-MICROPY_EVENT_POLL_HOOK
+mp_event_handle_nowait();
 #endif
     }
 
@@ -199,8 +199,8 @@ mp_obj_t Badger2040_halt(mp_obj_t self_in) {
 
     self->badger2040->update_button_states();
     while (self->badger2040->button_states() == 0) {
-#ifdef MICROPY_EVENT_POLL_HOOK
+#ifdef mp_event_handle_nowait
-MICROPY_EVENT_POLL_HOOK
+mp_event_handle_nowait();
 #endif
       self->badger2040->update_button_states();
     }

micropython/modules/jpegdec/jpegdec.cpp

@@ -84,8 +84,8 @@ int32_t jpegdec_seek_callback(JPEGFILE *jpeg, int32_t p) {
 }
 
 int JPEGDraw(JPEGDRAW *pDraw) {
-#ifdef MICROPY_EVENT_POLL_HOOK
+#ifdef mp_event_handle_nowait
-MICROPY_EVENT_POLL_HOOK
+mp_event_handle_nowait();
 #endif
     PicoGraphics *current_graphics = (PicoGraphics *)pDraw->pUser;
     // "pixel" is slow and clipped,

micropython/modules/pico_scroll/pico_scroll.cpp

@@ -110,7 +110,7 @@ mp_obj_t picoscroll_scroll_text(mp_uint_t n_args, const mp_obj_t *args) {
             self->scroll->clear();
             self->scroll->set_bitmap_1d((const char *)draw_buffer, draw_buffer_len, brightness, offset);
             self->scroll->update();
-    MICROPY_EVENT_POLL_HOOK
+    mp_event_handle_nowait();
             sleep_ms(delay_ms);
         }
 

micropython/modules/picographics/picographics.cpp

@@ -623,16 +623,16 @@ mp_obj_t ModPicoGraphics_update(mp_obj_t self_in) {
 */
 
     while(self->display->is_busy()) {
-    #ifdef MICROPY_EVENT_POLL_HOOK
+    #ifdef mp_event_handle_nowait
-    MICROPY_EVENT_POLL_HOOK
+    mp_event_handle_nowait();
     #endif
     }
 
     self->display->update(self->graphics);
 
     while(self->display->is_busy()) {
-    #ifdef MICROPY_EVENT_POLL_HOOK
+    #ifdef mp_event_handle_nowait
-    MICROPY_EVENT_POLL_HOOK
+    mp_event_handle_nowait();
     #endif
     }
 
@@ -647,8 +647,8 @@ mp_obj_t ModPicoGraphics_partial_update(size_t n_args, const mp_obj_t *args) {
     ModPicoGraphics_obj_t *self = MP_OBJ_TO_PTR2(args[ARG_self], ModPicoGraphics_obj_t);
 
     while(self->display->is_busy()) {
-    #ifdef MICROPY_EVENT_POLL_HOOK
+    #ifdef mp_event_handle_nowait
-    MICROPY_EVENT_POLL_HOOK
+    mp_event_handle_nowait();
     #endif
     }
 
@@ -660,8 +660,8 @@ mp_obj_t ModPicoGraphics_partial_update(size_t n_args, const mp_obj_t *args) {
     });
 
     while(self->display->is_busy()) {
-    #ifdef MICROPY_EVENT_POLL_HOOK
+    #ifdef mp_event_handle_nowait
-    MICROPY_EVENT_POLL_HOOK
+    mp_event_handle_nowait();
     #endif
     }
 

micropython/modules/pimoroni_i2c/pimoroni_i2c.cpp

@@ -8,7 +8,7 @@ using namespace pimoroni;
 extern "C" {
 #include "pimoroni_i2c.h"
 #include "py/mperrno.h"
-#include "extmod/machine_i2c.h"
+#include "extmod/modmachine.h"
 
 
 _PimoroniI2C_obj_t*  PimoroniI2C_from_machine_i2c_or_native(mp_obj_t i2c_obj) {

micropython/modules/pimoroni_i2c/pimoroni_i2c.h

@@ -1,6 +1,6 @@
 // Include MicroPython API.
 #include "py/runtime.h"
-#include "extmod/machine_i2c.h"
+#include "extmod/modmachine.h"
 #include "hardware/i2c.h"
 
 /***** Extern of Class Definition *****/

micropython/modules/pngdec/pngdec.cpp

@@ -118,8 +118,8 @@ void pngdec_open_helper(_PNG_obj_t *self) {
 }
 
 void PNGDraw(PNGDRAW *pDraw) {
-#ifdef MICROPY_EVENT_POLL_HOOK
+#ifdef mp_event_handle_nowait
-MICROPY_EVENT_POLL_HOOK
+mp_event_handle_nowait();
 #endif
     _PNG_decode_target *target = (_PNG_decode_target*)pDraw->pUser;
     PicoGraphics *current_graphics = (PicoGraphics *)target->target;

micropython/modules_py/modules_py.cmake

@@ -2,28 +2,28 @@ set(MODULES_DIR ${CMAKE_CURRENT_LIST_DIR})
 
 function (copy_module MODULE)
     add_custom_command(
-        OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/../modules/${MODULE}
+        OUTPUT ${MICROPY_PORT_DIR}/modules/${MODULE}
 
         COMMAND
-            cp ${MODULES_DIR}/${MODULE} ${CMAKE_CURRENT_BINARY_DIR}/../modules/${MODULE}
+            cp ${MODULES_DIR}/${MODULE} ${MICROPY_PORT_DIR}/modules/${MODULE}
 
         DEPENDS ${MODULES_DIR}/${MODULE}
     )
 
-    target_sources(usermod_modules_py INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/../modules/${MODULE})
+    target_sources(usermod_modules_py INTERFACE ${MICROPY_PORT_DIR}/modules/${MODULE})
 endfunction()
 
 function (genversion VERSION_FILE)
     add_custom_command(
-        OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/../modules/${VERSION_FILE}
+        OUTPUT ${MICROPY_PORT_DIR}/modules/${VERSION_FILE}
 
         COMMAND
-            bash ${MODULES_DIR}/genversion.sh ${CMAKE_CURRENT_BINARY_DIR}/../modules/${VERSION_FILE}
+            bash ${MODULES_DIR}/genversion.sh ${MICROPY_PORT_DIR}/modules/${VERSION_FILE}
 
         DEPENDS ${MODULES_DIR}/genversion.sh
     )
 
-    target_sources(usermod_modules_py INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/../modules/${VERSION_FILE})
+    target_sources(usermod_modules_py INTERFACE ${MICROPY_PORT_DIR}/modules/${VERSION_FILE})
 endfunction()
 
 # Create a dummy usermod to hang our .py copies from