CI: Move build steps to a bash script.

2023-11-16 12:03:59 +00:00 · 2023-11-16 12:03:59 +00:00 · 6154116662
commit 6154116662
--- a/.github/workflows/micropython.yml
+++ b/.github/workflows/micropython.yml
@ -8,62 +8,10 @@ on:

 env:
  MICROPYTHON_VERSION: v1.22.1
-  WORKFLOW_VERSION: v1

 jobs:
-  deps:
-    runs-on: ubuntu-20.04
-    name: Dependencies
-    steps:
-    - name: Workspace Cache
-      id: cache
-      uses: actions/cache@v3
-      with:
-        path: ${{runner.workspace}}
-        key: workspace-micropython-${{env.MICROPYTHON_VERSION}}-${{env.WORKFLOW_VERSION}}
-        restore-keys: |
-          workspace-micropython-${{env.MICROPYTHON_VERSION}}-${{env.WORKFLOW_VERSION}}
-
-    # Check out MicroPython
-    - name: Checkout MicroPython
-      if: steps.cache.outputs.cache-hit != 'true'
-      uses: actions/checkout@v3
-      with:
-        repository: micropython/micropython
-        ref: ${{env.MICROPYTHON_VERSION}}
-        submodules: false  # MicroPython submodules are hideously broken
-        path: micropython
-
-    # Check out MicroPython Libs
-    - name: Checkout MicroPython Libs
-      if: steps.cache.outputs.cache-hit != 'true'
-      uses: actions/checkout@v3
-      with:
-        repository: micropython/micropython-lib
-        path: micropython-lib
-
-    - name: Fetch Pico submodules
-      if: steps.cache.outputs.cache-hit != 'true'
-      shell: bash
-      working-directory: micropython/ports/rp2
-      run: |
-        git submodule update --init ../../lib/pico-sdk
-        git submodule update --init ../../lib/cyw43-driver
-        git submodule update --init ../../lib/lwip
-        git submodule update --init ../../lib/mbedtls
-        git submodule update --init ../../lib/micropython-lib
-        git submodule update --init ../../lib/tinyusb
-        git submodule update --init ../../lib/btstack
-
-    - name: Build mpy-cross
-      if: steps.cache.outputs.cache-hit != 'true'
-      shell: bash
-      working-directory: micropython/mpy-cross
-      run: make
-
  build:
-    needs: deps
-    name: ${{matrix.name}} (${{matrix.board}})
+    name: ${{ matrix.name }} (${{ matrix.board }})
    runs-on: ubuntu-20.04
    strategy:
      matrix:
@ -82,7 +30,6 @@ jobs:
          board: PIMORONI_TUFTY2040
        - name: enviro
          board: PICO_W_ENVIRO
-          patch: true
        - name: galactic_unicorn
          board: RPI_PICO_W
        - name: cosmic_unicorn
@ -91,93 +38,81 @@ jobs:
          board: RPI_PICO_W
        - name: inky_frame
          board: PICO_W_INKY
-          patch: true

    env:
      # MicroPython version will be contained in github.event.release.tag_name for releases
-      RELEASE_FILE: pimoroni-${{matrix.name}}-${{github.event.release.tag_name || github.sha}}-micropython
-      MICROPY_BOARD_DIR: "$GITHUB_WORKSPACE/pimoroni-pico-${{ github.sha }}/micropython/board/${{ matrix.BOARD }}"
-      USER_C_MODULES: "$GITHUB_WORKSPACE/pimoroni-pico-${{ github.sha }}/micropython/modules/micropython-${{matrix.name}}.cmake"
+      RELEASE_FILE: pimoroni-${{ matrix.name }}-${{ github.event.release.tag_name || github.sha }}-micropython
+      PIMORONI_PICO_DIR: "${{ github.workspace }}/pimoroni-pico-${{ github.sha }}"
+      MICROPY_BOARD_DIR: "${{ github.workspace }}/pimoroni-pico-${{ github.sha }}/micropython/board/${{ matrix.BOARD }}"
+      USER_C_MODULES: "${{ github.workspace }}/pimoroni-pico-${{ github.sha }}/micropython/modules/micropython-${{ matrix.name }}.cmake"
+      TAG_OR_SHA: ${{ github.event.release.tag_name || github.sha }}
+      MICROPY_BOARD: ${{ matrix.board }}
+      BOARD_NAME: ${{ matrix.name }}
+      BUILD_TOOLS: pimoroni-pico-${{ github.sha }}/ci/micropython.sh

    steps:
    - name: Compiler Cache
      uses: actions/cache@v3
      with:
        path: /home/runner/.ccache
-        key: ccache-micropython-${{matrix.name}}-${{github.ref}}-${{github.sha}}
+        key: ccache-micropython-${{ matrix.name }}-${{ github.ref }}-${{ github.sha }}
        restore-keys: |
-          ccache-micropython-${{matrix.name}}-${{github.ref}}
-          ccache-micropython-${{matrix.name}}-
+          ccache-micropython-${{ matrix.name }}-${{ github.ref }}
+          ccache-micropython-${{ matrix.name }}-
  
-    - name: Workspace Cache
-      uses: actions/cache@v3
-      with:
-        path: ${{runner.workspace}}
-        key: workspace-micropython-${{env.MICROPYTHON_VERSION}}-${{env.WORKFLOW_VERSION}}
-        restore-keys: |
-          workspace-micropython-${{env.MICROPYTHON_VERSION}}-${{env.WORKFLOW_VERSION}}
-
-    - name: Install Compiler & CCache
-      if: runner.os == 'Linux'
-      run: |
-        sudo apt update && sudo apt install ccache gcc-arm-none-eabi
-  
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
      with:
        submodules: true
        path: pimoroni-pico-${{ github.sha }}
-
-    - name: Set MicroPython Version Env Vars
-      shell: bash
+      
+    - name: Install Compiler & CCache
      run: |
-        echo "MICROPY_GIT_TAG=$MICROPYTHON_VERSION, ${{matrix.name}} ${{github.event.release.tag_name || github.sha}}" >> $GITHUB_ENV
-        echo "MICROPY_GIT_HASH=$MICROPYTHON_VERSION-${{github.event.release.tag_name || github.sha}}" >> $GITHUB_ENV
+        source $BUILD_TOOLS
+        build_deps
+      
+    - name: Checkout MicroPython & MicroPython Libs
+      run: |
+        source $BUILD_TOOLS
+        clone_micropython_lib
+        clone_micropython

    - name: "HACK: CMakeLists.txt Disable C++ Exceptions Patch"
      shell: bash
-      working-directory: micropython
-      run: git apply $GITHUB_WORKSPACE/pimoroni-pico-${{ github.sha }}/micropython/micropython_nano_specs.patch
+      run: |
+        source $BUILD_TOOLS
+        hack_patch_micropython_disable_exceptions

    - name: "HACK: Pico SDK Patch"
-      if: matrix.patch == true
      shell: bash
-      working-directory: micropython
      run: |
-        $GITHUB_WORKSPACE/pimoroni-pico-${{ github.sha }}/micropython/board/pico-sdk-patch.sh ${{matrix.board}}
+        source $BUILD_TOOLS
+        hack_patch_pico_sdk

    - name: Configure MicroPython
      shell: bash
-      working-directory: micropython/ports/rp2
      run: |
-        cmake -S . -B build-${{matrix.name}} -DPICO_BUILD_DOCS=0 -DUSER_C_MODULES=${{env.USER_C_MODULES}} -DMICROPY_BOARD_DIR=${{env.MICROPY_BOARD_DIR}} -DMICROPY_BOARD=${{matrix.board}} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+        source $BUILD_TOOLS
+        cmake_configure

-    - name: Build MicroPython  # Multiple simultaneous jobs trigger https://github.com/pimoroni/pimoroni-pico/issues/761
+    - name: Build MicroPython
      shell: bash
-      working-directory: micropython/ports/rp2
      run: |
-        ccache --zero-stats || true
-        cmake --build build-${{matrix.name}} -j 1
-        ccache --show-stats || true
-
-    - name: Rename .uf2 for artifact
-      shell: bash
-      working-directory: micropython/ports/rp2/build-${{matrix.name}}
-      run: |
-        cp firmware.uf2 $RELEASE_FILE.uf2
+        source $BUILD_TOOLS
+        cmake_build

    - name: Store .uf2 as artifact
      uses: actions/upload-artifact@v3
      with:
-        name: ${{env.RELEASE_FILE}}.uf2
-        path: micropython/ports/rp2/build-${{matrix.name}}/${{env.RELEASE_FILE}}.uf2
+        name: ${{ env.RELEASE_FILE }}.uf2
+        path: build-${{ matrix.name }}/${{ env.RELEASE_FILE }}.uf2

    - name: Upload .uf2
      if: github.event_name == 'release'
      uses: actions/upload-release-asset@v1
      env:
-        GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      with:
-        asset_path: micropython/ports/rp2/build-${{matrix.name}}/firmware.uf2
-        upload_url: ${{github.event.release.upload_url}}
-        asset_name: ${{env.RELEASE_FILE}}.uf2
+        asset_path: build-${{ matrix.name }}/firmware.uf2
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_name: ${{ env.RELEASE_FILE }}.uf2
        asset_content_type: application/octet-stream
--- a/ci/micropython.sh
+++ b/ci/micropython.sh
@ -0,0 +1,77 @@
+export TERM=${TERM:="xterm-256color"}
+
+function log_success {
+	echo -e "$(tput setaf 2)$1$(tput sgr0)"
+}
+
+function log_inform {
+	echo -e "$(tput setaf 6)$1$(tput sgr0)"
+}
+
+function log_warning {
+	echo -e "$(tput setaf 1)$1$(tput sgr0)"
+}
+
+function clone_micropython_lib {
+    git clone https://github.com/micropython/micropython-lib --depth=1
+}
+
+function clone_micropython {
+    log_inform "Using MicroPython $MICROPYTHON_VERSION"
+    git clone https://github.com/micropython/micropython --depth=1 --branch=$MICROPYTHON_VERSION
+    cd micropython
+    git submodule update --init lib/pico-sdk
+    git submodule update --init lib/cyw43-driver
+    git submodule update --init lib/lwip
+    git submodule update --init lib/mbedtls
+    git submodule update --init lib/micropython-lib
+    git submodule update --init lib/tinyusb
+    git submodule update --init lib/btstack
+    cd mpy-cross
+    make
+    cd ../../
+}
+
+function build_deps {
+    sudo apt update && sudo apt install ccache gcc-arm-none-eabi
+}
+
+function clone_pimoroni_pico {
+    git clone https://github.com/micropython/micropython --branch=$PIMORONI_PICO_SHA --depth=1
+}
+
+function micropython_version {
+    echo "MICROPY_GIT_TAG=\"$MICROPYTHON_VERSION, $BOARD_NAME $TAG_OR_SHA\"" >> $GITHUB_ENV
+    echo "MICROPY_GIT_HASH=\"$MICROPYTHON_VERSION-$TAG_OR_SHA\"" >> $GITHUB_ENV
+}
+
+function hack_patch_micropython_disable_exceptions {
+    cd micropython
+    git apply $PIMORONI_PICO_DIR/micropython/micropython_nano_specs.patch
+    cd ../
+}
+
+function hack_patch_pico_sdk {
+    # pico-sdk-patch.sh will apply the patch if it exists
+    cd micropython
+    $PIMORONI_PICO_DIR/micropython/board/pico-sdk-patch.sh $MICROPY_BOARD
+    cd ../
+}
+
+function cmake_configure {
+    cmake -S micropython/ports/rp2 -B build-$BOARD_NAME \
+    -DPICO_BUILD_DOCS=0 \
+    -DUSER_C_MODULES=$USER_C_MODULES \
+    -DMICROPY_BOARD_DIR=$MICROPY_BOARD_DIR \
+    -DMICROPY_BOARD=$MICROPY_BOARD \
+    -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+    -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+}
+
+function cmake_build {
+    ccache --zero-stats || true
+    cmake --build build-$BOARD_NAME -j 2
+    ccache --show-stats || true
+    cd build-$BOARD_NAME
+    cp firmware.uf2 $RELEASE_FILE.uf2
+}
--- a/micropython/modules_py/modules_py.cmake
+++ b/micropython/modules_py/modules_py.cmake
@ -2,28 +2,28 @@ set(MODULES_DIR ${CMAKE_CURRENT_LIST_DIR})

 function (copy_module MODULE)
    add_custom_command(
-        OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/../modules/${MODULE}
+        OUTPUT ${MICROPY_PORT_DIR}/modules/${MODULE}

        COMMAND
-            cp ${MODULES_DIR}/${MODULE} ${CMAKE_CURRENT_BINARY_DIR}/../modules/${MODULE}
+            cp ${MODULES_DIR}/${MODULE} ${MICROPY_PORT_DIR}/modules/${MODULE}

        DEPENDS ${MODULES_DIR}/${MODULE}
    )

-    target_sources(usermod_modules_py INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/../modules/${MODULE})
+    target_sources(usermod_modules_py INTERFACE ${MICROPY_PORT_DIR}/modules/${MODULE})
 endfunction()

 function (genversion VERSION_FILE)
    add_custom_command(
-        OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/../modules/${VERSION_FILE}
+        OUTPUT ${MICROPY_PORT_DIR}/modules/${VERSION_FILE}

        COMMAND
-            bash ${MODULES_DIR}/genversion.sh ${CMAKE_CURRENT_BINARY_DIR}/../modules/${VERSION_FILE}
+            bash ${MODULES_DIR}/genversion.sh ${MICROPY_PORT_DIR}/modules/${VERSION_FILE}

        DEPENDS ${MODULES_DIR}/genversion.sh
    )

-    target_sources(usermod_modules_py INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/../modules/${VERSION_FILE})
+    target_sources(usermod_modules_py INTERFACE ${MICROPY_PORT_DIR}/modules/${VERSION_FILE})
 endfunction()

 # Create a dummy usermod to hang our .py copies from