Implement automatic CDN publishing (#403)

.github/actions/publish-firmware/action.yml

@@ -0,0 +1,24 @@
+name: Publish Firmware
+
+inputs:
+  cdn_version:
+    description: The version that determines the path on the CDN
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: 'Download all-firmwares'
+      uses: actions/download-artifact@v4
+      with:
+        name: 'all-firmwares'
+        path: firmwares
+    - name: 'Install boto3'
+      shell: bash
+      run: pip install boto3
+    - name: 'Generate files'
+      shell: bash
+      run: version=`cat version.txt` && python Buildscripts/CDN/generate-files.py firmwares firmwares-cdn $version
+    - name: 'Upload files'
+      shell: bash
+      run: python Buildscripts/CDN/upload-files.py firmwares-cdn ${{ inputs.cdn_version }} ${{ env.CDN_ID }} ${{ env.CDN_TOKEN_NAME }} ${{ env.CDN_TOKEN_VALUE }}

.github/workflows/build-firmware.yml

@@ -3,6 +3,8 @@ on:
   push:
     branches:
       - main
+    tags:
+      - v*
   pull_request:
     types: [ opened, synchronize, reopened ]
 
@@ -57,8 +59,38 @@ jobs:
   Bundle:
     runs-on: ubuntu-latest
     needs: [ Build ]
-    if: (github.event_name == 'pull_request' && startsWith(github.head_ref, 'release'))
+    if: |
+      (github.event_name == 'push' && github.ref == 'refs/heads/main') ||
+      (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v'))
     steps:
       - uses: actions/checkout@v4
       - name: "Bundle"
         uses: ./.github/actions/bundle-firmware
+  PublishSnapshot:
+    runs-on: ubuntu-latest
+    needs: [ Bundle ]
+    if: (github.event_name == 'push' && github.ref == 'refs/heads/main')
+    steps:
+      - uses: actions/checkout@v4
+      - name: "Publish Snapshot"
+        env:
+          CDN_ID: ${{ secrets.CDN_ID }}
+          CDN_TOKEN_NAME: ${{ secrets.CDN_TOKEN_NAME }}
+          CDN_TOKEN_VALUE: ${{ secrets.CDN_TOKEN_VALUE }}
+        uses: ./.github/actions/publish-firmware
+        with:
+          cdn_version: snapshot
+  PublishRelease:
+    runs-on: ubuntu-latest
+    needs: [ Bundle ]
+    if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v'))
+    steps:
+      - uses: actions/checkout@v4
+      - name: "Publish Stable"
+        env:
+          CDN_ID: ${{ secrets.CDN_ID }}
+          CDN_TOKEN_NAME: ${{ secrets.CDN_TOKEN_NAME }}
+          CDN_TOKEN_VALUE: ${{ secrets.CDN_TOKEN_VALUE }}
+        uses: ./.github/actions/publish-firmware
+        with:
+          cdn_version: stable

Buildscripts/CDN/devices.properties

@@ -0,0 +1,164 @@
+[btt-panda-touch]
+vendor=BigTreeTech
+boardName=Panda Touch,K Touch
+incubating=false
+
+[cyd-2432s024c]
+vendor=CYD
+boardName=2432S024C
+incubating=false
+warningMessage=There currently is a known issue with the display driver.<br/>It will likely show artifacts.
+
+[cyd-2432s028r]
+vendor=CYD
+boardName=2432S028R
+incubating=false
+warningMessage=There are 3 hardware variants of this board. This build works on the original variant only ("v1").
+
+[cyd-2432s028rv3]
+vendor=CYD
+boardName=2432S028R v3
+incubating=false
+warningMessage=There are 3 hardware variants of this board. This build only supports board version 3.
+
+[cyd-2432s032c]
+vendor=CYD
+boardName=2432S032C
+incubating=false
+
+[cyd-4848s040c]
+vendor=CYD
+boardName=4848S040C
+incubating=false
+
+[cyd-8048s043c]
+vendor=CYD
+boardName=8048S043C
+incubating=false
+
+[cyd-e32r28t]
+vendor=CYD
+boardName=E32R28T
+incubating=false
+
+[cyd-e32r32p]
+vendor=CYD
+boardName=E32R32P
+incubating=false
+
+[cyd-jc2432w328c]
+vendor=CYD
+boardName=JC2432W328C
+incubating=false
+
+[cyd-jc8048w550c]
+vendor=CYD
+boardName=JC8048W550C
+incubating=false
+
+[elecrow-crowpanel-advance-28]
+vendor=Elecrow
+boardName=CrowPanel Advance 2.8"
+incubating=false
+
+[elecrow-crowpanel-advance-35]
+vendor=Elecrow
+boardName=CrowPanel Advance 3.5"
+incubating=false
+
+[elecrow-crowpanel-advance-50]
+vendor=Elecrow
+boardName=CrowPanel Advance 5"
+incubating=false
+
+[elecrow-crowpanel-basic-28]
+vendor=Elecrow
+boardName=CrowPanel Basic 2.8"
+incubating=false
+
+[elecrow-crowpanel-basic-35]
+vendor=Elecrow
+boardName=CrowPanel Basic 3.5"
+incubating=false
+
+[elecrow-crowpanel-basic-50]
+vendor=Elecrow
+boardName=CrowPanel Basic 5"
+incubating=false
+
+[lilygo-tdeck]
+vendor=LilyGO
+boardName=T-Deck,T-Deck Plus
+incubating=false
+infoMessage=If two serial devices are visible, try them both.<br/><br/>To put the device into bootloader mode: <br/>1. Press the trackball and then the reset button at the same time,<br/>2. Let go of the reset button, then the trackball.<br/><br/>When this website reports that flashing is finished, you likely have to press the reset button.
+
+[lilygo-tdisplay-s3]
+vendor=LilyGO
+boardName=T-Display S3
+incubating=false
+
+[lilygo-tdongle-s3]
+vendor=LilyGO
+boardName=T-Dongle S3
+incubating=true
+
+[lilygo-tlora-pager]
+vendor=LilyGO
+boardName=T-Lora Pager
+incubating=false
+
+[m5stack-cardputer]
+vendor=M5Stack
+boardName=Cardputer,Cardputer v1.1
+incubating=false
+
+[m5stack-cardputer-adv]
+vendor=M5Stack
+boardName=Cardputer Adv
+incubating=false
+
+[m5stack-core2]
+vendor=M5Stack
+boardName=Core2
+incubating=false
+
+[m5stack-cores3]
+vendor=M5Stack
+boardName=CoreS3
+incubating=false
+
+[m5stack-stickc-plus]
+vendor=M5Stack
+boardName=StickC Plus
+incubating=true
+
+[m5stack-stickc-plus2]
+vendor=M5Stack
+boardName=StickC Plus2
+incubating=true
+
+[unphone]
+vendor=unPhone
+boardName=unPhone
+incubating=false
+warningMessage=There is a power drain issue that slowly depletes the device when it\'s off. It lasts about 3 days.<br/>Completely depleting a battery can permanently decrease capacity. ?<br/><br/>This is a newly implemented device, so there might be other issues. Use at your own risk.<br/><br/>Put the device into bootloader mode by pressing the center nav button and reset for 2-3 seconds, then release reset, then release the nav button.<br/>After flashing is finished, press the reset button to reboot.
+
+[waveshare-s3-lcd-13]
+vendor=WaveShare
+boardName=S3 LCD 1.3"
+incubating=true
+
+[waveshare-s3-touch-lcd-43]
+vendor=WaveShare
+boardName=S3 Touch LCD 4.3"
+incubating=false
+
+[waveshare-s3-touch-lcd-128]
+vendor=WaveShare
+boardName=S3 Touch LCD 1.28"
+incubating=true
+
+[waveshare-s3-touch-lcd-147]
+vendor=WaveShare
+boardName=S3 Touch LCD 1.47"
+incubating=true

Buildscripts/CDN/generate-files.py

@@ -0,0 +1,198 @@
+import os
+import sys
+import configparser
+from dataclasses import dataclass, asdict
+import json
+import shutil
+
+verbose = False
+
+@dataclass
+class IndexEntry:
+    id: str
+    name: str
+    vendor: str
+    incubating: bool
+    warningMessage: str
+    infoMessage: str
+
+@dataclass
+class Manifest:
+    name: str
+    version: str
+    new_install_prompt_erase: str
+    funding_url: str
+    builds: list
+
+@dataclass
+class ManifestBuild:
+    chipFamily: str
+    parts: list
+
+@dataclass
+class ManifestBuildPart:
+    path: str
+    offset: int
+
+@dataclass
+class DeviceIndex:
+    version: str
+    devices: list
+
+if sys.platform == "win32":
+    shell_color_red = ""
+    shell_color_orange = ""
+    shell_color_reset = ""
+else:
+    shell_color_red = "\033[91m"
+    shell_color_orange = "\033[93m"
+    shell_color_reset = "\033[m"
+
+def print_warning(message):
+    print(f"{shell_color_orange}WARNING: {message}{shell_color_reset}")
+
+def print_error(message):
+    print(f"{shell_color_red}ERROR: {message}{shell_color_reset}")
+
+def print_help():
+    print("Usage: python generate-files.py [inPath] [outPath] [version]")
+    print("     inPath   path with the extracted release files")
+    print("     outPath  path where the CDN files will become available")
+    print("     version  technical version name (e.g. 1.2.0)")
+    print("Options:")
+    print("  --verbose                      Show extra console output")
+
+def exit_with_error(message):
+    print_error(message)
+    sys.exit(1)
+
+def read_properties_file(path):
+    config = configparser.RawConfigParser()
+    config.read(path)
+    return config
+
+def read_mapping_file():
+    mapping_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "devices.properties")
+    if not os.path.isfile(mapping_file_path):
+        exit_with_error(f"Mapping file not found: {mapping_file_path}")
+    return read_properties_file(mapping_file_path)
+
+def to_manifest_chip_name(name):
+    if name == "esp32":
+        return "ESP32"
+    elif name == "esp32s2":
+        return "ESP32-S2"
+    elif name == "esp32s3":
+        return "ESP32-S3"
+    elif name == "esp32c3":
+        return "ESP32-C3"
+    elif name == "esp32c5":
+        return "ESP32-C5"
+    elif name == "esp32c6":
+        return "ESP32-C6"
+    elif name == "esp32p4":
+        return "ESP32-P4"
+    else:
+        exit_with_error(f"to_manifest_chip_name() doesn't support {name} yet")
+        return ""
+
+
+def process_board(in_path: str, out_path: str, device_directory: str, device_id: str, device_mapping: configparser, version: str):
+    in_device_path = os.path.join(in_path, device_directory)
+    in_device_binaries_path = os.path.join(in_device_path, "Binaries")
+    assert os.path.isdir(in_device_binaries_path)
+    flasher_args_path = os.path.join(in_device_binaries_path, "flasher_args.json")
+    assert os.path.isfile(flasher_args_path)
+    with open(flasher_args_path) as json_data:
+        flasher_args = json.load(json_data)
+        json_data.close()
+        flash_files = flasher_args["flash_files"]
+        manifest = Manifest(
+            name=f"Tactility for {device_mapping["vendor"]} {device_mapping["boardName"]}",
+            version=version,
+            new_install_prompt_erase="true",
+            funding_url="https://github.com/sponsors/ByteWelder",
+            builds=[
+                ManifestBuild(
+                    chipFamily=to_manifest_chip_name(flasher_args["extra_esptool_args"]["chip"]),
+                    parts=[]
+                )
+            ]
+        )
+        for offset in flash_files:
+            flash_file_entry = flash_files[offset]
+            flash_file_entry_name = os.path.basename(flash_file_entry)
+            in_flash_file_path = os.path.join(in_device_binaries_path, flash_file_entry)
+            out_flash_file_name = f"{device_id}-{flash_file_entry_name}"
+            out_flash_file_path = os.path.join(out_path, out_flash_file_name)
+            if verbose:
+                print(f"Copying {in_flash_file_path} -> {out_flash_file_path}")
+            shutil.copy(in_flash_file_path, out_flash_file_path)
+            manifest.builds[0].parts.append(
+                ManifestBuildPart(
+                    path=out_flash_file_name,
+                    offset=int(offset, 16)
+                )
+            )
+
+        json_manifest_path = os.path.join(out_path, f"{device_id}.json")
+        with open(json_manifest_path, 'w') as json_manifest_file:
+            json.dump(asdict(manifest), json_manifest_file, indent=2)
+            json_manifest_file.close()
+
+def main(in_path: str, out_path: str, version: str):
+    if not os.path.exists(in_path):
+        exit_with_error(f"Input path not found: {in_path}")
+    if os.path.exists(out_path):
+        shutil.rmtree(out_path)
+    os.mkdir(out_path)
+    mapping = read_mapping_file()
+    device_directories = os.listdir(in_path)
+    device_index = DeviceIndex(version, [])
+    for device_directory in device_directories:
+        if not device_directory.endswith("-symbols"):
+            device_id = device_directory[10:]
+            if device_id not in mapping.sections():
+                exit_with_error(f"Mapping for {device_id} not found in mapping file")
+            device_properties = mapping[device_id]
+            process_board(in_path, out_path, device_directory, device_id, device_properties, version)
+            if "warningMessage" in device_properties.keys():
+                warning_message = device_properties["warningMessage"]
+            else:
+                warning_message = None
+            if "infoMessage" in device_properties.keys():
+                info_message = device_properties["infoMessage"]
+            else:
+                info_message = None
+            if "incubating" in device_properties.keys():
+                incubating = device_properties["incubating"].lower() == 'true'
+            else:
+                incubating = False
+            board_names = device_properties["boardName"].split(',')
+            for board_name in board_names:
+                device_index.devices.append(asdict(IndexEntry(
+                    id=device_id,
+                    name=board_name,
+                    vendor=device_properties["vendor"],
+                    incubating=incubating,
+                    warningMessage=warning_message,
+                    infoMessage=info_message
+                )))
+    index_file_path = os.path.join(out_path, "index.json")
+    with open(index_file_path, "w") as index_file:
+        json.dump(asdict(device_index), index_file, indent=2)
+        index_file.close()
+
+if __name__ == "__main__":
+    print("Tactility CDN File Generator")
+    if "--help" in sys.argv:
+        print_help()
+        sys.exit()
+    # Argument validation
+    if len(sys.argv) < 4:
+        print_help()
+        sys.exit()
+    if "--verbose" in sys.argv:
+        verbose = True
+        sys.argv.remove("--verbose")
+    main(in_path=sys.argv[1], out_path=sys.argv[2], version=sys.argv[3])

Buildscripts/CDN/upload-files.py

@@ -0,0 +1,76 @@
+import os
+import sys
+import boto3
+
+verbose = False
+
+if sys.platform == "win32":
+    shell_color_red = ""
+    shell_color_orange = ""
+    shell_color_reset = ""
+else:
+    shell_color_red = "\033[91m"
+    shell_color_orange = "\033[93m"
+    shell_color_reset = "\033[m"
+
+def print_warning(message):
+    print(f"{shell_color_orange}WARNING: {message}{shell_color_reset}")
+
+def print_error(message):
+    print(f"{shell_color_red}ERROR: {message}{shell_color_reset}")
+
+def print_help():
+    print("Usage: python upload-files.py [path] [version] [cloudflareAccountId] [cloudflareTokenName] [cloudflareTokenValue]")
+    print("")
+    print("Options:")
+    print("  --verbose                      Show extra console output")
+    print("  --index-only                   Upload only index.json")
+
+def exit_with_error(message):
+    print_error(message)
+    sys.exit(1)
+
+def main(path: str, version: str, cloudflare_account_id, cloudflare_token_name: str, cloudflare_token_value: str, index_only: bool):
+    if not os.path.exists(path):
+        exit_with_error(f"Path not found: {path}")
+    s3 = boto3.client(
+        service_name="s3",
+        endpoint_url=f"https://{cloudflare_account_id}.r2.cloudflarestorage.com",
+        aws_access_key_id=cloudflare_token_name,
+        aws_secret_access_key=cloudflare_token_value,
+        region_name="auto"
+    )
+    files_to_upload = os.listdir(path)
+    counter = 1
+    total = len(files_to_upload)
+    for file_name in files_to_upload:
+        if not index_only or file_name == 'index.json':
+            object_path = f"firmware/{version}/{file_name}"
+            print(f"[{counter}/{total}] Uploading {file_name} to {object_path}")
+            file_path = os.path.join(path, file_name)
+            try:
+                s3.upload_file(file_path, "tactility", object_path)
+            except Exception as e:
+                exit_with_error(f"Failed to upload {file_name}: {str(e)}")
+            counter += 1
+
+if __name__ == "__main__":
+    print("Tactility CDN Uploader")
+    if "--help" in sys.argv:
+        print_help()
+        sys.exit()
+    # Argument validation
+    if len(sys.argv) < 6:
+        print_help()
+        sys.exit()
+    if "--verbose" in sys.argv:
+        verbose = True
+        sys.argv.remove("--verbose")
+    main(
+        path=sys.argv[1],
+        version=sys.argv[2],
+        cloudflare_account_id=sys.argv[3],
+        cloudflare_token_name=sys.argv[4],
+        cloudflare_token_value=sys.argv[5],
+        index_only="--index-only" in sys.argv
+    )

version.txt

@@ -1 +1 @@
-0.6.0
+0.7.0-dev