Merge pull request #878 from pimoroni/ci/tooling

CI: Move some workflow steps into ci/micropython.sh
2024-01-19 10:32:20 +00:00 · 2024-01-19 10:32:20 +00:00 · 392d75b00d
commit 392d75b00d
--- a/.github/workflows/micropython.yml
+++ b/.github/workflows/micropython.yml
@ -7,63 +7,11 @@ on:
    types: [created]

 env:
-  MICROPYTHON_VERSION: v1.21.0
-  WORKFLOW_VERSION: v1
+  MICROPYTHON_VERSION: v1.22.1

 jobs:
-  deps:
-    runs-on: ubuntu-20.04
-    name: Dependencies
-    steps:
-    - name: Workspace Cache
-      id: cache
-      uses: actions/cache@v3
-      with:
-        path: ${{runner.workspace}}
-        key: workspace-micropython-${{env.MICROPYTHON_VERSION}}-${{env.WORKFLOW_VERSION}}
-        restore-keys: |
-          workspace-micropython-${{env.MICROPYTHON_VERSION}}-${{env.WORKFLOW_VERSION}}
-
-    # Check out MicroPython
-    - name: Checkout MicroPython
-      if: steps.cache.outputs.cache-hit != 'true'
-      uses: actions/checkout@v3
-      with:
-        repository: micropython/micropython
-        ref: ${{env.MICROPYTHON_VERSION}}
-        submodules: false  # MicroPython submodules are hideously broken
-        path: micropython
-
-    # Check out MicroPython Libs
-    - name: Checkout MicroPython Libs
-      if: steps.cache.outputs.cache-hit != 'true'
-      uses: actions/checkout@v3
-      with:
-        repository: micropython/micropython-lib
-        path: micropython-lib
-
-    - name: Fetch Pico submodules
-      if: steps.cache.outputs.cache-hit != 'true'
-      shell: bash
-      working-directory: micropython/ports/rp2
-      run: |
-        git submodule update --init ../../lib/pico-sdk
-        git submodule update --init ../../lib/cyw43-driver
-        git submodule update --init ../../lib/lwip
-        git submodule update --init ../../lib/mbedtls
-        git submodule update --init ../../lib/micropython-lib
-        git submodule update --init ../../lib/tinyusb
-        git submodule update --init ../../lib/btstack
-
-    - name: Build mpy-cross
-      if: steps.cache.outputs.cache-hit != 'true'
-      shell: bash
-      working-directory: micropython/mpy-cross
-      run: make
-
  build:
-    needs: deps
-    name: ${{matrix.name}} (${{matrix.board}})
+    name: ${{ matrix.name }} (${{ matrix.board }})
    runs-on: ubuntu-20.04
    strategy:
      matrix:
@ -82,7 +30,6 @@ jobs:
          board: PIMORONI_TUFTY2040
        - name: enviro
          board: PICO_W_ENVIRO
-          patch: true
        - name: galactic_unicorn
          board: RPI_PICO_W
        - name: cosmic_unicorn
@ -91,93 +38,91 @@ jobs:
          board: RPI_PICO_W
        - name: inky_frame
          board: PICO_W_INKY
-          patch: true

    env:
      # MicroPython version will be contained in github.event.release.tag_name for releases
-      RELEASE_FILE: pimoroni-${{matrix.name}}-${{github.event.release.tag_name || github.sha}}-micropython
-      MICROPY_BOARD_DIR: "$GITHUB_WORKSPACE/pimoroni-pico-${{ github.sha }}/micropython/board/${{ matrix.BOARD }}"
-      USER_C_MODULES: "$GITHUB_WORKSPACE/pimoroni-pico-${{ github.sha }}/micropython/modules/micropython-${{matrix.name}}.cmake"
+      RELEASE_FILE: pimoroni-${{ matrix.name }}-${{ github.event.release.tag_name || github.sha }}-micropython
+      PIMORONI_PICO_DIR: "${{ github.workspace }}/pimoroni-pico-${{ github.sha }}"
+      MICROPY_BOARD_DIR: "${{ github.workspace }}/pimoroni-pico-${{ github.sha }}/micropython/board/${{ matrix.BOARD }}"
+      USER_C_MODULES: "${{ github.workspace }}/pimoroni-pico-${{ github.sha }}/micropython/modules/micropython-${{ matrix.name }}.cmake"
+      TAG_OR_SHA: ${{ github.event.release.tag_name || github.sha }}
+      MICROPY_BOARD: ${{ matrix.board }}
+      BOARD_NAME: ${{ matrix.name }}
+      BUILD_TOOLS: pimoroni-pico-${{ github.sha }}/ci/micropython.sh

    steps:
    - name: Compiler Cache
      uses: actions/cache@v3
      with:
        path: /home/runner/.ccache
-        key: ccache-micropython-${{matrix.name}}-${{github.ref}}-${{github.sha}}
+        key: ccache-micropython-${{ matrix.name }}-${{ github.ref }}-${{ github.sha }}
        restore-keys: |
-          ccache-micropython-${{matrix.name}}-${{github.ref}}
-          ccache-micropython-${{matrix.name}}-
-  
-    - name: Workspace Cache
-      uses: actions/cache@v3
-      with:
-        path: ${{runner.workspace}}
-        key: workspace-micropython-${{env.MICROPYTHON_VERSION}}-${{env.WORKFLOW_VERSION}}
-        restore-keys: |
-          workspace-micropython-${{env.MICROPYTHON_VERSION}}-${{env.WORKFLOW_VERSION}}
+          ccache-micropython-${{ matrix.name }}-${{ github.ref }}
+          ccache-micropython-${{ matrix.name }}-

-    - name: Install Compiler & CCache
-      if: runner.os == 'Linux'
-      run: |
-        sudo apt update && sudo apt install ccache gcc-arm-none-eabi
-  
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
      with:
        submodules: true
        path: pimoroni-pico-${{ github.sha }}

-    - name: Set MicroPython Version Env Vars
-      shell: bash
+    - name: Install Arm GNU Toolchain (arm-none-eabi-gcc)
+      uses: carlosperate/arm-none-eabi-gcc-action@v1
+      with:
+        release: '9-2020-q2'
+
+    - name: Install CCache
      run: |
-        echo "MICROPY_GIT_TAG=$MICROPYTHON_VERSION, ${{matrix.name}} ${{github.event.release.tag_name || github.sha}}" >> $GITHUB_ENV
-        echo "MICROPY_GIT_HASH=$MICROPYTHON_VERSION-${{github.event.release.tag_name || github.sha}}" >> $GITHUB_ENV
+        source $BUILD_TOOLS
+        apt_install_build_deps
+      
+    - name: Checkout MicroPython & Submodules
+      run: |
+        source $BUILD_TOOLS
+        micropython_clone
+
+    - name: Build MPY Cross
+      run: |
+        source $BUILD_TOOLS
+        micropython_build_mpy_cross

    - name: "HACK: CMakeLists.txt Disable C++ Exceptions Patch"
      shell: bash
-      working-directory: micropython
-      run: git apply $GITHUB_WORKSPACE/pimoroni-pico-${{ github.sha }}/micropython/micropython_nano_specs.patch
+      run: |
+        source $BUILD_TOOLS
+        hack_patch_micropython_disable_exceptions

    - name: "HACK: Pico SDK Patch"
-      if: matrix.patch == true
      shell: bash
-      working-directory: micropython
      run: |
-        $GITHUB_WORKSPACE/pimoroni-pico-${{ github.sha }}/micropython/board/pico-sdk-patch.sh ${{matrix.board}}
+        source $BUILD_TOOLS
+        hack_patch_pico_sdk

    - name: Configure MicroPython
      shell: bash
-      working-directory: micropython/ports/rp2
      run: |
-        cmake -S . -B build-${{matrix.name}} -DPICO_BUILD_DOCS=0 -DUSER_C_MODULES=${{env.USER_C_MODULES}} -DMICROPY_BOARD_DIR=${{env.MICROPY_BOARD_DIR}} -DMICROPY_BOARD=${{matrix.board}} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+        source $BUILD_TOOLS
+        micropython_version
+        cmake_configure

-    - name: Build MicroPython  # Multiple simultaneous jobs trigger https://github.com/pimoroni/pimoroni-pico/issues/761
+    - name: Build MicroPython
      shell: bash
-      working-directory: micropython/ports/rp2
      run: |
-        ccache --zero-stats || true
-        cmake --build build-${{matrix.name}} -j 1
-        ccache --show-stats || true
-
-    - name: Rename .uf2 for artifact
-      shell: bash
-      working-directory: micropython/ports/rp2/build-${{matrix.name}}
-      run: |
-        cp firmware.uf2 $RELEASE_FILE.uf2
+        source $BUILD_TOOLS
+        cmake_build

    - name: Store .uf2 as artifact
      uses: actions/upload-artifact@v3
      with:
-        name: ${{env.RELEASE_FILE}}.uf2
-        path: micropython/ports/rp2/build-${{matrix.name}}/${{env.RELEASE_FILE}}.uf2
+        name: ${{ env.RELEASE_FILE }}.uf2
+        path: build-${{ matrix.name }}/${{ env.RELEASE_FILE }}.uf2

    - name: Upload .uf2
      if: github.event_name == 'release'
      uses: actions/upload-release-asset@v1
      env:
-        GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      with:
-        asset_path: micropython/ports/rp2/build-${{matrix.name}}/firmware.uf2
-        upload_url: ${{github.event.release.upload_url}}
-        asset_name: ${{env.RELEASE_FILE}}.uf2
+        asset_path: build-${{ matrix.name }}/firmware.uf2
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_name: ${{ env.RELEASE_FILE }}.uf2
        asset_content_type: application/octet-stream
--- a/ci/micropython.sh
+++ b/ci/micropython.sh
@ -0,0 +1,75 @@
+export TERM=${TERM:="xterm-256color"}
+
+function log_success {
+	echo -e "$(tput setaf 2)$1$(tput sgr0)"
+}
+
+function log_inform {
+	echo -e "$(tput setaf 6)$1$(tput sgr0)"
+}
+
+function log_warning {
+	echo -e "$(tput setaf 1)$1$(tput sgr0)"
+}
+
+function micropython_clone {
+    log_inform "Using MicroPython $MICROPYTHON_VERSION"
+    git clone https://github.com/micropython/micropython --depth=1 --branch=$MICROPYTHON_VERSION
+    cd micropython
+    git submodule update --init lib/pico-sdk
+    git submodule update --init lib/cyw43-driver
+    git submodule update --init lib/lwip
+    git submodule update --init lib/mbedtls
+    git submodule update --init lib/micropython-lib
+    git submodule update --init lib/tinyusb
+    git submodule update --init lib/btstack
+    cd ../
+}
+
+function micropython_build_mpy_cross {
+    cd micropython/mpy-cross
+    ccache --zero-stats || true
+    CROSS_COMPILE="ccache " make
+    ccache --show-stats || true
+    cd ../../
+}
+
+function apt_install_build_deps {
+    sudo apt update && sudo apt install ccache
+}
+
+function micropython_version {
+    echo "MICROPY_GIT_TAG=$MICROPYTHON_VERSION, $BOARD_NAME $TAG_OR_SHA" >> $GITHUB_ENV
+    echo "MICROPY_GIT_HASH=$MICROPYTHON_VERSION-$TAG_OR_SHA" >> $GITHUB_ENV
+}
+
+function hack_patch_micropython_disable_exceptions {
+    cd micropython
+    git apply $PIMORONI_PICO_DIR/micropython/micropython_nano_specs.patch
+    cd ../
+}
+
+function hack_patch_pico_sdk {
+    # pico-sdk-patch.sh will apply the patch if it exists
+    cd micropython
+    $PIMORONI_PICO_DIR/micropython/board/pico-sdk-patch.sh $MICROPY_BOARD
+    cd ../
+}
+
+function cmake_configure {
+    cmake -S micropython/ports/rp2 -B build-$BOARD_NAME \
+    -DPICO_BUILD_DOCS=0 \
+    -DUSER_C_MODULES=$USER_C_MODULES \
+    -DMICROPY_BOARD_DIR=$MICROPY_BOARD_DIR \
+    -DMICROPY_BOARD=$MICROPY_BOARD \
+    -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+    -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+}
+
+function cmake_build {
+    ccache --zero-stats || true
+    cmake --build build-$BOARD_NAME -j 2
+    ccache --show-stats || true
+    cd build-$BOARD_NAME
+    cp firmware.uf2 $RELEASE_FILE.uf2
+}
--- a/micropython/board/pico-sdk-patch.sh
+++ b/micropython/board/pico-sdk-patch.sh
@ -7,4 +7,6 @@ if [[ -f "$FIXUP_DIR/$BOARD/pico_sdk.patch" ]]; then
    echo "Applying pico_sdk.patch"
    cd $MPY_DIR/lib/pico-sdk
    git apply "$FIXUP_DIR/$BOARD/pico_sdk.patch"
+else
+    echo "Skipping patch. $FIXUP_DIR/$BOARD/pico_sdk.patch not found!"
 fi
--- a/micropython/micropython_nano_specs.patch
+++ b/micropython/micropython_nano_specs.patch
@ -1,11 +1,11 @@
 diff --git a/ports/rp2/CMakeLists.txt b/ports/rp2/CMakeLists.txt
-index 094031c6852a..5f268414c08f 100644
+index 281b0c3bc..7e04bb549 100644
 --- a/ports/rp2/CMakeLists.txt
 +++ b/ports/rp2/CMakeLists.txt
-@@ -374,6 +374,15 @@ target_compile_options(${MICROPY_TARGET} PRIVATE
- target_link_options(${MICROPY_TARGET} PRIVATE
-     -Wl,--defsym=__micropy_c_heap_size__=${MICROPY_C_HEAP_SIZE}
+@@ -453,6 +453,16 @@ target_link_options(${MICROPY_TARGET} PRIVATE
+     -Wl,--wrap=dcd_event_handler
 )
+ 
 +# Do not include stack unwinding & exception handling for C++ user modules
 +target_compile_definitions(usermod INTERFACE PICO_CXX_ENABLE_EXCEPTIONS=0)
 +target_compile_options(usermod INTERFACE $<$<COMPILE_LANGUAGE:CXX>:
@ -15,6 +15,7 @@ index 094031c6852a..5f268414c08f 100644
 +    -fno-use-cxa-atexit
 +>)
 +target_link_options(usermod INTERFACE -specs=nano.specs)
- 
+
 set_source_files_properties(
     ${PICO_SDK_PATH}/src/rp2_common/pico_double/double_math.c
+     ${PICO_SDK_PATH}/src/rp2_common/pico_float/float_math.c
--- a/micropython/modules/badger2040/badger2040.cpp
+++ b/micropython/modules/badger2040/badger2040.cpp
@ -122,8 +122,8 @@ mp_obj_t Badger2040_update(mp_obj_t self_in) {
    _Badger2040_obj_t *self = MP_OBJ_TO_PTR2(self_in, _Badger2040_obj_t);

    while(self->badger2040->is_busy()) {
-#ifdef MICROPY_EVENT_POLL_HOOK
-MICROPY_EVENT_POLL_HOOK
+#ifdef mp_event_handle_nowait
+mp_event_handle_nowait();
 #endif
    }

@ -133,8 +133,8 @@ MICROPY_EVENT_POLL_HOOK
    // Ensure blocking for the minimum amount of time
    // in cases where "is_busy" is unreliable.
    while(self->badger2040->is_busy() || absolute_time_diff_us(get_absolute_time(), t_end) > 0) {
-#ifdef MICROPY_EVENT_POLL_HOOK
-MICROPY_EVENT_POLL_HOOK
+#ifdef mp_event_handle_nowait
+mp_event_handle_nowait();
 #endif
    }

@ -166,8 +166,8 @@ mp_obj_t Badger2040_partial_update(size_t n_args, const mp_obj_t *pos_args, mp_m


    while(self->badger2040->is_busy()) {
-#ifdef MICROPY_EVENT_POLL_HOOK
-MICROPY_EVENT_POLL_HOOK
+#ifdef mp_event_handle_nowait
+mp_event_handle_nowait();
 #endif
    }

@ -177,8 +177,8 @@ MICROPY_EVENT_POLL_HOOK
    // Ensure blocking for the minimum amount of time
    // in cases where "is_busy" is unreliable.
    while(self->badger2040->is_busy() || absolute_time_diff_us(get_absolute_time(), t_end) > 0) {
-#ifdef MICROPY_EVENT_POLL_HOOK
-MICROPY_EVENT_POLL_HOOK
+#ifdef mp_event_handle_nowait
+mp_event_handle_nowait();
 #endif
    }

@ -199,8 +199,8 @@ mp_obj_t Badger2040_halt(mp_obj_t self_in) {

    self->badger2040->update_button_states();
    while (self->badger2040->button_states() == 0) {
-#ifdef MICROPY_EVENT_POLL_HOOK
-MICROPY_EVENT_POLL_HOOK
+#ifdef mp_event_handle_nowait
+mp_event_handle_nowait();
 #endif
      self->badger2040->update_button_states();
    }
--- a/micropython/modules/jpegdec/jpegdec.cpp
+++ b/micropython/modules/jpegdec/jpegdec.cpp
@ -84,8 +84,8 @@ int32_t jpegdec_seek_callback(JPEGFILE *jpeg, int32_t p) {
 }

 int JPEGDraw(JPEGDRAW *pDraw) {
-#ifdef MICROPY_EVENT_POLL_HOOK
-MICROPY_EVENT_POLL_HOOK
+#ifdef mp_event_handle_nowait
+mp_event_handle_nowait();
 #endif
    PicoGraphics *current_graphics = (PicoGraphics *)pDraw->pUser;
    // "pixel" is slow and clipped,
--- a/micropython/modules/pico_scroll/pico_scroll.cpp
+++ b/micropython/modules/pico_scroll/pico_scroll.cpp
@ -110,7 +110,7 @@ mp_obj_t picoscroll_scroll_text(mp_uint_t n_args, const mp_obj_t *args) {
            self->scroll->clear();
            self->scroll->set_bitmap_1d((const char *)draw_buffer, draw_buffer_len, brightness, offset);
            self->scroll->update();
-    MICROPY_EVENT_POLL_HOOK
+    mp_event_handle_nowait();
            sleep_ms(delay_ms);
        }

--- a/micropython/modules/picographics/picographics.cpp
+++ b/micropython/modules/picographics/picographics.cpp
@ -623,16 +623,16 @@ mp_obj_t ModPicoGraphics_update(mp_obj_t self_in) {
 */

    while(self->display->is_busy()) {
-    #ifdef MICROPY_EVENT_POLL_HOOK
-    MICROPY_EVENT_POLL_HOOK
+    #ifdef mp_event_handle_nowait
+    mp_event_handle_nowait();
    #endif
    }

    self->display->update(self->graphics);

    while(self->display->is_busy()) {
-    #ifdef MICROPY_EVENT_POLL_HOOK
-    MICROPY_EVENT_POLL_HOOK
+    #ifdef mp_event_handle_nowait
+    mp_event_handle_nowait();
    #endif
    }

@ -647,8 +647,8 @@ mp_obj_t ModPicoGraphics_partial_update(size_t n_args, const mp_obj_t *args) {
    ModPicoGraphics_obj_t *self = MP_OBJ_TO_PTR2(args[ARG_self], ModPicoGraphics_obj_t);

    while(self->display->is_busy()) {
-    #ifdef MICROPY_EVENT_POLL_HOOK
-    MICROPY_EVENT_POLL_HOOK
+    #ifdef mp_event_handle_nowait
+    mp_event_handle_nowait();
    #endif
    }

@ -660,8 +660,8 @@ mp_obj_t ModPicoGraphics_partial_update(size_t n_args, const mp_obj_t *args) {
    });

    while(self->display->is_busy()) {
-    #ifdef MICROPY_EVENT_POLL_HOOK
-    MICROPY_EVENT_POLL_HOOK
+    #ifdef mp_event_handle_nowait
+    mp_event_handle_nowait();
    #endif
    }

--- a/micropython/modules/pimoroni_i2c/pimoroni_i2c.cpp
+++ b/micropython/modules/pimoroni_i2c/pimoroni_i2c.cpp
@ -8,7 +8,7 @@ using namespace pimoroni;
 extern "C" {
 #include "pimoroni_i2c.h"
 #include "py/mperrno.h"
-#include "extmod/machine_i2c.h"
+#include "extmod/modmachine.h"


 _PimoroniI2C_obj_t*  PimoroniI2C_from_machine_i2c_or_native(mp_obj_t i2c_obj) {
--- a/micropython/modules/pimoroni_i2c/pimoroni_i2c.h
+++ b/micropython/modules/pimoroni_i2c/pimoroni_i2c.h
@ -1,6 +1,6 @@
 // Include MicroPython API.
 #include "py/runtime.h"
-#include "extmod/machine_i2c.h"
+#include "extmod/modmachine.h"
 #include "hardware/i2c.h"

 /***** Extern of Class Definition *****/
--- a/micropython/modules/pngdec/pngdec.cpp
+++ b/micropython/modules/pngdec/pngdec.cpp
@ -118,8 +118,8 @@ void pngdec_open_helper(_PNG_obj_t *self) {
 }

 void PNGDraw(PNGDRAW *pDraw) {
-#ifdef MICROPY_EVENT_POLL_HOOK
-MICROPY_EVENT_POLL_HOOK
+#ifdef mp_event_handle_nowait
+mp_event_handle_nowait();
 #endif
    _PNG_decode_target *target = (_PNG_decode_target*)pDraw->pUser;
    PicoGraphics *current_graphics = (PicoGraphics *)target->target;
--- a/micropython/modules_py/modules_py.cmake
+++ b/micropython/modules_py/modules_py.cmake
@ -2,28 +2,28 @@ set(MODULES_DIR ${CMAKE_CURRENT_LIST_DIR})

 function (copy_module MODULE)
    add_custom_command(
-        OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/../modules/${MODULE}
+        OUTPUT ${MICROPY_PORT_DIR}/modules/${MODULE}

        COMMAND
-            cp ${MODULES_DIR}/${MODULE} ${CMAKE_CURRENT_BINARY_DIR}/../modules/${MODULE}
+            cp ${MODULES_DIR}/${MODULE} ${MICROPY_PORT_DIR}/modules/${MODULE}

        DEPENDS ${MODULES_DIR}/${MODULE}
    )

-    target_sources(usermod_modules_py INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/../modules/${MODULE})
+    target_sources(usermod_modules_py INTERFACE ${MICROPY_PORT_DIR}/modules/${MODULE})
 endfunction()

 function (genversion VERSION_FILE)
    add_custom_command(
-        OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/../modules/${VERSION_FILE}
+        OUTPUT ${MICROPY_PORT_DIR}/modules/${VERSION_FILE}

        COMMAND
-            bash ${MODULES_DIR}/genversion.sh ${CMAKE_CURRENT_BINARY_DIR}/../modules/${VERSION_FILE}
+            bash ${MODULES_DIR}/genversion.sh ${MICROPY_PORT_DIR}/modules/${VERSION_FILE}

        DEPENDS ${MODULES_DIR}/genversion.sh
    )

-    target_sources(usermod_modules_py INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/../modules/${VERSION_FILE})
+    target_sources(usermod_modules_py INTERFACE ${MICROPY_PORT_DIR}/modules/${VERSION_FILE})
 endfunction()

 # Create a dummy usermod to hang our .py copies from