From 0f82730a78184eedca1505a94373ee8be9f1a8bb Mon Sep 17 00:00:00 2001 From: Andy Shinn Date: Wed, 21 Apr 2021 17:41:36 -0500 Subject: [PATCH] Release engineering improvements (#1844) * version set from package.json and release bin names * support direnv virtualenv * versioned PlatformIO environment * matrix support for parralel CI * gather artifacts * release on tagging * minor scripts formatting --- .envrc | 1 + .github/workflows/wled-ci.yml | 67 +++++++++++++++++++++++++++++++--- .gitignore | 1 + pio-scripts/gzip-firmware.py | 23 ------------ pio-scripts/name-firmware.py | 34 ----------------- pio-scripts/output_bins.py | 69 +++++++++++++++++++++++++++++++++++ pio-scripts/set_version.py | 8 ++++ platformio.ini | 17 +++++---- requirements.in | 1 + requirements.txt | 54 +++++++++++++++++++++++++++ wled00/wled.h | 17 ++++++--- 11 files changed, 216 insertions(+), 76 deletions(-) create mode 100644 .envrc delete mode 100644 pio-scripts/gzip-firmware.py delete mode 100644 pio-scripts/name-firmware.py create mode 100644 pio-scripts/output_bins.py create mode 100644 pio-scripts/set_version.py create mode 100644 requirements.in create mode 100644 requirements.txt diff --git a/.envrc b/.envrc new file mode 100644 index 000000000..83dab5d19 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +layout python-venv python3 diff --git a/.github/workflows/wled-ci.yml b/.github/workflows/wled-ci.yml index 4158dd7ac..c28446e61 100644 --- a/.github/workflows/wled-ci.yml +++ b/.github/workflows/wled-ci.yml @@ -3,10 +3,37 @@ name: PlatformIO CI on: [push, pull_request] jobs: - build: + get_default_envs: + name: Gather Environments runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Cache pip + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - uses: actions/setup-python@v2 + - name: Install PlatformIO + run: pip install -r requirements.txt + - name: Get default environments + id: envs + run: | + echo "::set-output name=environments::$(pio project config --json-output | jq -cr '.[0][1][0][1]')" + outputs: + environments: ${{ steps.envs.outputs.environments }} + + build: + name: Build Enviornments + runs-on: ubuntu-latest + needs: get_default_envs + strategy: + matrix: + environment: ${{ fromJSON(needs.get_default_envs.outputs.environments) }} steps: - uses: actions/checkout@v2 - name: Cache pip @@ -24,8 +51,36 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 - name: Install PlatformIO - run: | - python -m pip install --upgrade pip - pip install --upgrade platformio - - name: Run PlatformIO - run: pio run \ No newline at end of file + run: pip install -r requirements.txt + - name: Build firmware + env: + WLED_RELEASE: True + run: pio run -e ${{ matrix.environment }} + - uses: actions/upload-artifact@v2 + with: + name: firmware-${{ matrix.environment }} + path: | + build_output/firmware/*.bin + build_output/firmware/*.gz + - uses: actions/upload-artifact@v2 + if: startsWith(github.ref, 'refs/tags/') + with: + name: firmware-release + path: build_output/release/*.bin + release: + name: Create Release + runs-on: ubuntu-latest + needs: [get_default_envs, build] + if: startsWith(github.ref, 'refs/tags/') + steps: + - uses: actions/download-artifact@v2 + with: + name: firmware-release + - name: Create draft release + uses: softprops/action-gh-release@v1 + with: + draft: True + files: | + *.bin + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 7c0dd1e5b..02e648b88 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ .clang-format node_modules .idea +.direnv diff --git a/pio-scripts/gzip-firmware.py b/pio-scripts/gzip-firmware.py deleted file mode 100644 index 2d0283011..000000000 --- a/pio-scripts/gzip-firmware.py +++ /dev/null @@ -1,23 +0,0 @@ -Import('env') -import os -import shutil -import gzip - -OUTPUT_DIR = "build_output{}".format(os.path.sep) - -def bin_gzip(source, target, env): - variant = str(target[0]).split(os.path.sep)[2] - - # create string with location and file names based on variant - bin_file = "{}firmware{}{}.bin".format(OUTPUT_DIR, os.path.sep, variant) - gzip_file = "{}firmware{}{}.bin.gz".format(OUTPUT_DIR, os.path.sep, variant) - - # check if new target files exist and remove if necessary - if os.path.isfile(gzip_file): os.remove(gzip_file) - - # write gzip firmware file - with open(bin_file,"rb") as fp: - with gzip.open(gzip_file, "wb", compresslevel = 9) as f: - shutil.copyfileobj(fp, f) - -env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", [bin_gzip]) diff --git a/pio-scripts/name-firmware.py b/pio-scripts/name-firmware.py deleted file mode 100644 index 90ea9867f..000000000 --- a/pio-scripts/name-firmware.py +++ /dev/null @@ -1,34 +0,0 @@ -Import('env') -import os -import shutil - -OUTPUT_DIR = "build_output{}".format(os.path.sep) - -def bin_rename_copy(source, target, env): - variant = str(target[0]).split(os.path.sep)[2] - - # check if output directories exist and create if necessary - if not os.path.isdir(OUTPUT_DIR): - os.mkdir(OUTPUT_DIR) - - for d in ['firmware', 'map']: - if not os.path.isdir("{}{}".format(OUTPUT_DIR, d)): - os.mkdir("{}{}".format(OUTPUT_DIR, d)) - - # create string with location and file names based on variant - map_file = "{}map{}{}.map".format(OUTPUT_DIR, os.path.sep, variant) - bin_file = "{}firmware{}{}.bin".format(OUTPUT_DIR, os.path.sep, variant) - - # check if new target files exist and remove if necessary - for f in [map_file, bin_file]: - if os.path.isfile(f): - os.remove(f) - - # copy firmware.bin to firmware/.bin - shutil.copy(str(target[0]), bin_file) - - # copy firmware.map to map/.map - if os.path.isfile("firmware.map"): - shutil.move("firmware.map", map_file) - -env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", [bin_rename_copy]) diff --git a/pio-scripts/output_bins.py b/pio-scripts/output_bins.py new file mode 100644 index 000000000..5f9268fb1 --- /dev/null +++ b/pio-scripts/output_bins.py @@ -0,0 +1,69 @@ +Import('env') +import os +import shutil +import gzip + +OUTPUT_DIR = "build_output{}".format(os.path.sep) + +def _get_cpp_define_value(env, define): + define_list = [item[-1] for item in env["CPPDEFINES"] if item[0] == define] + + if define_list: + return define_list[0] + + return None + +def _create_dirs(dirs=["firmware", "map"]): + # check if output directories exist and create if necessary + if not os.path.isdir(OUTPUT_DIR): + os.mkdir(OUTPUT_DIR) + + for d in dirs: + if not os.path.isdir("{}{}".format(OUTPUT_DIR, d)): + os.mkdir("{}{}".format(OUTPUT_DIR, d)) + +def bin_rename_copy(source, target, env): + _create_dirs() + variant = env["PIOENV"] + + # create string with location and file names based on variant + map_file = "{}map{}{}.map".format(OUTPUT_DIR, os.path.sep, variant) + bin_file = "{}firmware{}{}.bin".format(OUTPUT_DIR, os.path.sep, variant) + + release_name = _get_cpp_define_value(env, "WLED_RELEASE_NAME") + + if release_name and os.getenv("WLED_RELEASE"): + _create_dirs(["release"]) + version = _get_cpp_define_value(env, "WLED_VERSION") + release_file = "{}release{}WLED_{}_{}.bin".format(OUTPUT_DIR, os.path.sep, version, release_name) + shutil.copy(str(target[0]), release_file) + + # check if new target files exist and remove if necessary + for f in [map_file, bin_file]: + if os.path.isfile(f): + os.remove(f) + + # copy firmware.bin to firmware/.bin + shutil.copy(str(target[0]), bin_file) + + # copy firmware.map to map/.map + if os.path.isfile("firmware.map"): + shutil.move("firmware.map", map_file) + +def bin_gzip(source, target, env): + _create_dirs() + variant = env["PIOENV"] + + # create string with location and file names based on variant + bin_file = "{}firmware{}{}.bin".format(OUTPUT_DIR, os.path.sep, variant) + gzip_file = "{}firmware{}{}.bin.gz".format(OUTPUT_DIR, os.path.sep, variant) + + # check if new target files exist and remove if necessary + if os.path.isfile(gzip_file): os.remove(gzip_file) + + # write gzip firmware file + with open(bin_file,"rb") as fp: + with gzip.open(gzip_file, "wb", compresslevel = 9) as f: + shutil.copyfileobj(fp, f) + +env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", [bin_rename_copy, bin_gzip]) diff --git a/pio-scripts/set_version.py b/pio-scripts/set_version.py new file mode 100644 index 000000000..1d8e076ea --- /dev/null +++ b/pio-scripts/set_version.py @@ -0,0 +1,8 @@ +Import('env') +import json + +PACKAGE_FILE = "package.json" + +with open(PACKAGE_FILE, "r") as package: + version = json.load(package)["version"] + env.Append(BUILD_FLAGS=[f"-DWLED_VERSION={version}"]) diff --git a/platformio.ini b/platformio.ini index ba2e13ca1..2fd697028 100644 --- a/platformio.ini +++ b/platformio.ini @@ -151,10 +151,11 @@ build_flags = -g -DCONFIG_LITTLEFS_FOR_IDF_3_2 [scripts_defaults] -extra_scripts = pio-scripts/name-firmware.py - pio-scripts/gzip-firmware.py - pio-scripts/strip-floats.py - pio-scripts/user_config_copy.py +extra_scripts = + pre:pio-scripts/set_version.py + post:pio-scripts/output_bins.py + post:pio-scripts/strip-floats.py + pre:pio-scripts/user_config_copy.py # ------------------------------------------------------------------------------ # COMMON SETTINGS: @@ -213,7 +214,7 @@ platform = ${common.platform_wled_default} platform_packages = ${common.platform_packages} board_build.ldscript = ${common.ldscript_4m1m} build_unflags = ${common.build_unflags} -build_flags = ${common.build_flags_esp8266} +build_flags = ${common.build_flags_esp8266} -D WLED_RELEASE_NAME=ESP8266 [env:esp01_1m_full] board = esp01_1m @@ -221,7 +222,7 @@ platform = ${common.platform_wled_default} platform_packages = ${common.platform_packages} board_build.ldscript = ${common.ldscript_1m128k} build_unflags = ${common.build_unflags} -build_flags = ${common.build_flags_esp8266} -D WLED_DISABLE_OTA +build_flags = ${common.build_flags_esp8266} -D WLED_RELEASE_NAME=ESP01 -D WLED_DISABLE_OTA [env:esp07] board = esp07 @@ -261,7 +262,7 @@ build_flags = ${common.build_flags_esp8266} -D LEDPIN=1 -D WLED_DISABLE_INFRARED board = esp32dev platform = espressif32@3.2 build_unflags = ${common.build_unflags} -build_flags = ${common.build_flags_esp32} +build_flags = ${common.build_flags_esp32} -D WLED_RELEASE_NAME=ESP32 lib_ignore = ESPAsyncTCP ESPAsyncUDP @@ -271,7 +272,7 @@ board = esp32-poe platform = espressif32@3.2 upload_speed = 921600 build_unflags = ${common.build_unflags} -build_flags = ${common.build_flags_esp32} -D RLYPIN=-1 -D WLED_USE_ETHERNET -D BTNPIN=-1 +build_flags = ${common.build_flags_esp32} -D WLED_RELEASE_NAME=ESP32_Ethernet -D RLYPIN=-1 -D WLED_USE_ETHERNET -D BTNPIN=-1 lib_ignore = ESPAsyncTCP ESPAsyncUDP diff --git a/requirements.in b/requirements.in new file mode 100644 index 000000000..7c715125f --- /dev/null +++ b/requirements.in @@ -0,0 +1 @@ +platformio diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..6fc16f0fa --- /dev/null +++ b/requirements.txt @@ -0,0 +1,54 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile +# +aiofiles==0.6.0 + # via platformio +ajsonrpc==1.1.0 + # via platformio +bottle==0.12.19 + # via platformio +certifi==2020.12.5 + # via requests +chardet==4.0.0 + # via requests +click==7.1.2 + # via + # platformio + # uvicorn +colorama==0.4.4 + # via platformio +h11==0.12.0 + # via + # uvicorn + # wsproto +idna==2.10 + # via requests +ifaddr==0.1.7 + # via zeroconf +marshmallow==3.11.1 + # via platformio +platformio==5.1.1 + # via -r requirements.in +pyelftools==0.27 + # via platformio +pyserial==3.5 + # via platformio +requests==2.25.1 + # via platformio +semantic-version==2.8.5 + # via platformio +starlette==0.14.2 + # via platformio +tabulate==0.8.9 + # via platformio +urllib3==1.26.4 + # via requests +uvicorn==0.13.4 + # via platformio +wsproto==1.0.0 + # via platformio +zeroconf==0.28.8 + # via platformio diff --git a/wled00/wled.h b/wled00/wled.h index 8391e4250..7b178ba38 100644 --- a/wled00/wled.h +++ b/wled00/wled.h @@ -51,7 +51,7 @@ //This is generally a terrible idea, but improves boot success on boards with a 3.3v regulator + cap setup that can't provide 400mA peaks //#define WLED_DISABLE_BROWNOUT_DET -// Library inclusions. +// Library inclusions. #include #ifdef ESP8266 #include @@ -171,8 +171,15 @@ # define _INIT_N(x) UNPACK x #endif +#define STRINGIFY(X) #X +#define TOSTRING(X) STRINGIFY(X) + +#ifndef WLED_VERSION + #define WLED_VERSION "dev" +#endif + // Global Variable definitions -WLED_GLOBAL char versionString[] _INIT("0.12.0"); +WLED_GLOBAL char versionString[] _INIT(TOSTRING(WLED_VERSION)); #define WLED_CODENAME "Hikari" // AP and OTA default passwords (for maximum security change them!) @@ -203,7 +210,7 @@ WLED_GLOBAL int8_t irPin _INIT(4); WLED_GLOBAL int8_t irPin _INIT(IRPIN); #endif -//WLED_GLOBAL byte presetToApply _INIT(0); +//WLED_GLOBAL byte presetToApply _INIT(0); WLED_GLOBAL char ntpServerName[33] _INIT("0.wled.pool.ntp.org"); // NTP server to use @@ -224,7 +231,7 @@ WLED_GLOBAL bool noWifiSleep _INIT(false); // disabling WLED_GLOBAL int ethernetType _INIT(WLED_ETH_DEFAULT); // ethernet board type #else WLED_GLOBAL int ethernetType _INIT(WLED_ETH_NONE); // use none for ethernet board type if default not defined - #endif + #endif #endif // LED CONFIG @@ -548,7 +555,7 @@ WLED_GLOBAL bool e131NewData _INIT(false); // led fx library object WLED_GLOBAL BusManager busses _INIT(BusManager()); WLED_GLOBAL WS2812FX strip _INIT(WS2812FX()); -WLED_GLOBAL BusConfig* busConfigs[WLED_MAX_BUSSES] _INIT({nullptr}); //temporary, to remember values from network callback until after +WLED_GLOBAL BusConfig* busConfigs[WLED_MAX_BUSSES] _INIT({nullptr}); //temporary, to remember values from network callback until after WLED_GLOBAL bool doInitBusses _INIT(false); // Usermod manager