From 569cce38fa7038473a4b8cc5bc254e3b72b3ce55 Mon Sep 17 00:00:00 2001 From: Ken Van Hoeylandt Date: Fri, 31 Oct 2025 23:39:41 +0100 Subject: [PATCH] Implement automatic CDN publishing (#403) --- .github/actions/publish-firmware/action.yml | 24 +++ .github/workflows/build-firmware.yml | 34 +++- Buildscripts/CDN/devices.properties | 164 ++++++++++++++++ Buildscripts/CDN/generate-files.py | 198 ++++++++++++++++++++ Buildscripts/CDN/upload-files.py | 76 ++++++++ version.txt | 2 +- 6 files changed, 496 insertions(+), 2 deletions(-) create mode 100644 .github/actions/publish-firmware/action.yml create mode 100644 Buildscripts/CDN/devices.properties create mode 100644 Buildscripts/CDN/generate-files.py create mode 100644 Buildscripts/CDN/upload-files.py diff --git a/.github/actions/publish-firmware/action.yml b/.github/actions/publish-firmware/action.yml new file mode 100644 index 00000000..b7128fcd --- /dev/null +++ b/.github/actions/publish-firmware/action.yml @@ -0,0 +1,24 @@ +name: Publish Firmware + +inputs: + cdn_version: + description: The version that determines the path on the CDN + required: true + +runs: + using: 'composite' + steps: + - name: 'Download all-firmwares' + uses: actions/download-artifact@v4 + with: + name: 'all-firmwares' + path: firmwares + - name: 'Install boto3' + shell: bash + run: pip install boto3 + - name: 'Generate files' + shell: bash + run: version=`cat version.txt` && python Buildscripts/CDN/generate-files.py firmwares firmwares-cdn $version + - name: 'Upload files' + shell: bash + run: python Buildscripts/CDN/upload-files.py firmwares-cdn ${{ inputs.cdn_version }} ${{ env.CDN_ID }} ${{ env.CDN_TOKEN_NAME }} ${{ env.CDN_TOKEN_VALUE }} \ No newline at end of file diff --git a/.github/workflows/build-firmware.yml b/.github/workflows/build-firmware.yml index 80582123..99c49d10 100644 --- a/.github/workflows/build-firmware.yml +++ b/.github/workflows/build-firmware.yml @@ -3,6 +3,8 @@ on: push: branches: - main + tags: + - v* pull_request: types: [ opened, synchronize, reopened ] @@ -57,8 +59,38 @@ jobs: Bundle: runs-on: ubuntu-latest needs: [ Build ] - if: (github.event_name == 'pull_request' && startsWith(github.head_ref, 'release')) + if: | + (github.event_name == 'push' && github.ref == 'refs/heads/main') || + (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) steps: - uses: actions/checkout@v4 - name: "Bundle" uses: ./.github/actions/bundle-firmware + PublishSnapshot: + runs-on: ubuntu-latest + needs: [ Bundle ] + if: (github.event_name == 'push' && github.ref == 'refs/heads/main') + steps: + - uses: actions/checkout@v4 + - name: "Publish Snapshot" + env: + CDN_ID: ${{ secrets.CDN_ID }} + CDN_TOKEN_NAME: ${{ secrets.CDN_TOKEN_NAME }} + CDN_TOKEN_VALUE: ${{ secrets.CDN_TOKEN_VALUE }} + uses: ./.github/actions/publish-firmware + with: + cdn_version: snapshot + PublishRelease: + runs-on: ubuntu-latest + needs: [ Bundle ] + if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) + steps: + - uses: actions/checkout@v4 + - name: "Publish Stable" + env: + CDN_ID: ${{ secrets.CDN_ID }} + CDN_TOKEN_NAME: ${{ secrets.CDN_TOKEN_NAME }} + CDN_TOKEN_VALUE: ${{ secrets.CDN_TOKEN_VALUE }} + uses: ./.github/actions/publish-firmware + with: + cdn_version: stable diff --git a/Buildscripts/CDN/devices.properties b/Buildscripts/CDN/devices.properties new file mode 100644 index 00000000..3ce37b97 --- /dev/null +++ b/Buildscripts/CDN/devices.properties @@ -0,0 +1,164 @@ +[btt-panda-touch] +vendor=BigTreeTech +boardName=Panda Touch,K Touch +incubating=false + +[cyd-2432s024c] +vendor=CYD +boardName=2432S024C +incubating=false +warningMessage=There currently is a known issue with the display driver.
It will likely show artifacts. + +[cyd-2432s028r] +vendor=CYD +boardName=2432S028R +incubating=false +warningMessage=There are 3 hardware variants of this board. This build works on the original variant only ("v1"). + +[cyd-2432s028rv3] +vendor=CYD +boardName=2432S028R v3 +incubating=false +warningMessage=There are 3 hardware variants of this board. This build only supports board version 3. + +[cyd-2432s032c] +vendor=CYD +boardName=2432S032C +incubating=false + +[cyd-4848s040c] +vendor=CYD +boardName=4848S040C +incubating=false + +[cyd-8048s043c] +vendor=CYD +boardName=8048S043C +incubating=false + +[cyd-e32r28t] +vendor=CYD +boardName=E32R28T +incubating=false + +[cyd-e32r32p] +vendor=CYD +boardName=E32R32P +incubating=false + +[cyd-jc2432w328c] +vendor=CYD +boardName=JC2432W328C +incubating=false + +[cyd-jc8048w550c] +vendor=CYD +boardName=JC8048W550C +incubating=false + +[elecrow-crowpanel-advance-28] +vendor=Elecrow +boardName=CrowPanel Advance 2.8" +incubating=false + +[elecrow-crowpanel-advance-35] +vendor=Elecrow +boardName=CrowPanel Advance 3.5" +incubating=false + +[elecrow-crowpanel-advance-50] +vendor=Elecrow +boardName=CrowPanel Advance 5" +incubating=false + +[elecrow-crowpanel-basic-28] +vendor=Elecrow +boardName=CrowPanel Basic 2.8" +incubating=false + +[elecrow-crowpanel-basic-35] +vendor=Elecrow +boardName=CrowPanel Basic 3.5" +incubating=false + +[elecrow-crowpanel-basic-50] +vendor=Elecrow +boardName=CrowPanel Basic 5" +incubating=false + +[lilygo-tdeck] +vendor=LilyGO +boardName=T-Deck,T-Deck Plus +incubating=false +infoMessage=If two serial devices are visible, try them both.

To put the device into bootloader mode:
1. Press the trackball and then the reset button at the same time,
2. Let go of the reset button, then the trackball.

When this website reports that flashing is finished, you likely have to press the reset button. + +[lilygo-tdisplay-s3] +vendor=LilyGO +boardName=T-Display S3 +incubating=false + +[lilygo-tdongle-s3] +vendor=LilyGO +boardName=T-Dongle S3 +incubating=true + +[lilygo-tlora-pager] +vendor=LilyGO +boardName=T-Lora Pager +incubating=false + +[m5stack-cardputer] +vendor=M5Stack +boardName=Cardputer,Cardputer v1.1 +incubating=false + +[m5stack-cardputer-adv] +vendor=M5Stack +boardName=Cardputer Adv +incubating=false + +[m5stack-core2] +vendor=M5Stack +boardName=Core2 +incubating=false + +[m5stack-cores3] +vendor=M5Stack +boardName=CoreS3 +incubating=false + +[m5stack-stickc-plus] +vendor=M5Stack +boardName=StickC Plus +incubating=true + +[m5stack-stickc-plus2] +vendor=M5Stack +boardName=StickC Plus2 +incubating=true + +[unphone] +vendor=unPhone +boardName=unPhone +incubating=false +warningMessage=There is a power drain issue that slowly depletes the device when it\'s off. It lasts about 3 days.
Completely depleting a battery can permanently decrease capacity. ?

This is a newly implemented device, so there might be other issues. Use at your own risk.

Put the device into bootloader mode by pressing the center nav button and reset for 2-3 seconds, then release reset, then release the nav button.
After flashing is finished, press the reset button to reboot. + +[waveshare-s3-lcd-13] +vendor=WaveShare +boardName=S3 LCD 1.3" +incubating=true + +[waveshare-s3-touch-lcd-43] +vendor=WaveShare +boardName=S3 Touch LCD 4.3" +incubating=false + +[waveshare-s3-touch-lcd-128] +vendor=WaveShare +boardName=S3 Touch LCD 1.28" +incubating=true + +[waveshare-s3-touch-lcd-147] +vendor=WaveShare +boardName=S3 Touch LCD 1.47" +incubating=true diff --git a/Buildscripts/CDN/generate-files.py b/Buildscripts/CDN/generate-files.py new file mode 100644 index 00000000..f8123c4c --- /dev/null +++ b/Buildscripts/CDN/generate-files.py @@ -0,0 +1,198 @@ +import os +import sys +import configparser +from dataclasses import dataclass, asdict +import json +import shutil + +verbose = False + +@dataclass +class IndexEntry: + id: str + name: str + vendor: str + incubating: bool + warningMessage: str + infoMessage: str + +@dataclass +class Manifest: + name: str + version: str + new_install_prompt_erase: str + funding_url: str + builds: list + +@dataclass +class ManifestBuild: + chipFamily: str + parts: list + +@dataclass +class ManifestBuildPart: + path: str + offset: int + +@dataclass +class DeviceIndex: + version: str + devices: list + +if sys.platform == "win32": + shell_color_red = "" + shell_color_orange = "" + shell_color_reset = "" +else: + shell_color_red = "\033[91m" + shell_color_orange = "\033[93m" + shell_color_reset = "\033[m" + +def print_warning(message): + print(f"{shell_color_orange}WARNING: {message}{shell_color_reset}") + +def print_error(message): + print(f"{shell_color_red}ERROR: {message}{shell_color_reset}") + +def print_help(): + print("Usage: python generate-files.py [inPath] [outPath] [version]") + print(" inPath path with the extracted release files") + print(" outPath path where the CDN files will become available") + print(" version technical version name (e.g. 1.2.0)") + print("Options:") + print(" --verbose Show extra console output") + +def exit_with_error(message): + print_error(message) + sys.exit(1) + +def read_properties_file(path): + config = configparser.RawConfigParser() + config.read(path) + return config + +def read_mapping_file(): + mapping_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "devices.properties") + if not os.path.isfile(mapping_file_path): + exit_with_error(f"Mapping file not found: {mapping_file_path}") + return read_properties_file(mapping_file_path) + +def to_manifest_chip_name(name): + if name == "esp32": + return "ESP32" + elif name == "esp32s2": + return "ESP32-S2" + elif name == "esp32s3": + return "ESP32-S3" + elif name == "esp32c3": + return "ESP32-C3" + elif name == "esp32c5": + return "ESP32-C5" + elif name == "esp32c6": + return "ESP32-C6" + elif name == "esp32p4": + return "ESP32-P4" + else: + exit_with_error(f"to_manifest_chip_name() doesn't support {name} yet") + return "" + + +def process_board(in_path: str, out_path: str, device_directory: str, device_id: str, device_mapping: configparser, version: str): + in_device_path = os.path.join(in_path, device_directory) + in_device_binaries_path = os.path.join(in_device_path, "Binaries") + assert os.path.isdir(in_device_binaries_path) + flasher_args_path = os.path.join(in_device_binaries_path, "flasher_args.json") + assert os.path.isfile(flasher_args_path) + with open(flasher_args_path) as json_data: + flasher_args = json.load(json_data) + json_data.close() + flash_files = flasher_args["flash_files"] + manifest = Manifest( + name=f"Tactility for {device_mapping["vendor"]} {device_mapping["boardName"]}", + version=version, + new_install_prompt_erase="true", + funding_url="https://github.com/sponsors/ByteWelder", + builds=[ + ManifestBuild( + chipFamily=to_manifest_chip_name(flasher_args["extra_esptool_args"]["chip"]), + parts=[] + ) + ] + ) + for offset in flash_files: + flash_file_entry = flash_files[offset] + flash_file_entry_name = os.path.basename(flash_file_entry) + in_flash_file_path = os.path.join(in_device_binaries_path, flash_file_entry) + out_flash_file_name = f"{device_id}-{flash_file_entry_name}" + out_flash_file_path = os.path.join(out_path, out_flash_file_name) + if verbose: + print(f"Copying {in_flash_file_path} -> {out_flash_file_path}") + shutil.copy(in_flash_file_path, out_flash_file_path) + manifest.builds[0].parts.append( + ManifestBuildPart( + path=out_flash_file_name, + offset=int(offset, 16) + ) + ) + + json_manifest_path = os.path.join(out_path, f"{device_id}.json") + with open(json_manifest_path, 'w') as json_manifest_file: + json.dump(asdict(manifest), json_manifest_file, indent=2) + json_manifest_file.close() + +def main(in_path: str, out_path: str, version: str): + if not os.path.exists(in_path): + exit_with_error(f"Input path not found: {in_path}") + if os.path.exists(out_path): + shutil.rmtree(out_path) + os.mkdir(out_path) + mapping = read_mapping_file() + device_directories = os.listdir(in_path) + device_index = DeviceIndex(version, []) + for device_directory in device_directories: + if not device_directory.endswith("-symbols"): + device_id = device_directory[10:] + if device_id not in mapping.sections(): + exit_with_error(f"Mapping for {device_id} not found in mapping file") + device_properties = mapping[device_id] + process_board(in_path, out_path, device_directory, device_id, device_properties, version) + if "warningMessage" in device_properties.keys(): + warning_message = device_properties["warningMessage"] + else: + warning_message = None + if "infoMessage" in device_properties.keys(): + info_message = device_properties["infoMessage"] + else: + info_message = None + if "incubating" in device_properties.keys(): + incubating = device_properties["incubating"].lower() == 'true' + else: + incubating = False + board_names = device_properties["boardName"].split(',') + for board_name in board_names: + device_index.devices.append(asdict(IndexEntry( + id=device_id, + name=board_name, + vendor=device_properties["vendor"], + incubating=incubating, + warningMessage=warning_message, + infoMessage=info_message + ))) + index_file_path = os.path.join(out_path, "index.json") + with open(index_file_path, "w") as index_file: + json.dump(asdict(device_index), index_file, indent=2) + index_file.close() + +if __name__ == "__main__": + print("Tactility CDN File Generator") + if "--help" in sys.argv: + print_help() + sys.exit() + # Argument validation + if len(sys.argv) < 4: + print_help() + sys.exit() + if "--verbose" in sys.argv: + verbose = True + sys.argv.remove("--verbose") + main(in_path=sys.argv[1], out_path=sys.argv[2], version=sys.argv[3]) \ No newline at end of file diff --git a/Buildscripts/CDN/upload-files.py b/Buildscripts/CDN/upload-files.py new file mode 100644 index 00000000..09b36285 --- /dev/null +++ b/Buildscripts/CDN/upload-files.py @@ -0,0 +1,76 @@ +import os +import sys +import boto3 + +verbose = False + +if sys.platform == "win32": + shell_color_red = "" + shell_color_orange = "" + shell_color_reset = "" +else: + shell_color_red = "\033[91m" + shell_color_orange = "\033[93m" + shell_color_reset = "\033[m" + +def print_warning(message): + print(f"{shell_color_orange}WARNING: {message}{shell_color_reset}") + +def print_error(message): + print(f"{shell_color_red}ERROR: {message}{shell_color_reset}") + +def print_help(): + print("Usage: python upload-files.py [path] [version] [cloudflareAccountId] [cloudflareTokenName] [cloudflareTokenValue]") + print("") + print("Options:") + print(" --verbose Show extra console output") + print(" --index-only Upload only index.json") + +def exit_with_error(message): + print_error(message) + sys.exit(1) + +def main(path: str, version: str, cloudflare_account_id, cloudflare_token_name: str, cloudflare_token_value: str, index_only: bool): + if not os.path.exists(path): + exit_with_error(f"Path not found: {path}") + s3 = boto3.client( + service_name="s3", + endpoint_url=f"https://{cloudflare_account_id}.r2.cloudflarestorage.com", + aws_access_key_id=cloudflare_token_name, + aws_secret_access_key=cloudflare_token_value, + region_name="auto" + ) + files_to_upload = os.listdir(path) + counter = 1 + total = len(files_to_upload) + for file_name in files_to_upload: + if not index_only or file_name == 'index.json': + object_path = f"firmware/{version}/{file_name}" + print(f"[{counter}/{total}] Uploading {file_name} to {object_path}") + file_path = os.path.join(path, file_name) + try: + s3.upload_file(file_path, "tactility", object_path) + except Exception as e: + exit_with_error(f"Failed to upload {file_name}: {str(e)}") + counter += 1 + +if __name__ == "__main__": + print("Tactility CDN Uploader") + if "--help" in sys.argv: + print_help() + sys.exit() + # Argument validation + if len(sys.argv) < 6: + print_help() + sys.exit() + if "--verbose" in sys.argv: + verbose = True + sys.argv.remove("--verbose") + main( + path=sys.argv[1], + version=sys.argv[2], + cloudflare_account_id=sys.argv[3], + cloudflare_token_name=sys.argv[4], + cloudflare_token_value=sys.argv[5], + index_only="--index-only" in sys.argv + ) diff --git a/version.txt b/version.txt index 09a3acfa..9400325a 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.6.0 \ No newline at end of file +0.7.0-dev \ No newline at end of file