Ken Van Hoeylandt 0042ce6d32
Fix SDK (#487)
* **New Features**
  * Added a full SDK integration test suite with a minimal sample app and automated multi-platform build/run flows.
  * Added a new CLI build tool to orchestrate SDK download, build, packaging, and device install/run.

* **Documentation**
  * Added typography design token suggestion to ideas.

* **Chores**
  * Updated CI to run SDK integration prep and test steps.
  * Adjusted build artifact paths, license mappings, and repository ignore rules (including .tactility).
2026-02-06 22:50:13 +01:00

713 lines
27 KiB
Python

import configparser
import json
import os
import re
import shutil
import sys
import subprocess
import time
import urllib.request
import zipfile
import requests
import tarfile
from urllib.parse import urlparse
ttbuild_path = ".tactility"
ttbuild_version = "3.3.0"
ttbuild_cdn = "https://cdn.tactilityproject.org"
ttbuild_sdk_json_validity = 3600 # seconds
ttport = 6666
verbose = False
use_local_sdk = False
local_base_path = None
http_timeout_seconds = 10
shell_color_red = "\033[91m"
shell_color_orange = "\033[93m"
shell_color_green = "\033[32m"
shell_color_purple = "\033[35m"
shell_color_cyan = "\033[36m"
shell_color_reset = "\033[m"
def print_help():
print("Usage: python tactility.py [action] [options]")
print("")
print("Actions:")
print(" build [platform] Build the app. Optionally specify a platform.")
print(" Supported platforms are lower case. Example: esp32s3")
print(" Supported platforms are read from manifest.properties")
print(" clean Clean the build folders")
print(" clearcache Clear the SDK cache")
print(" updateself Update this tool")
print(" run [ip] Run the application")
print(" install [ip] Install the application")
print(" uninstall [ip] Uninstall the application")
print(" bir [ip] [platform] Build, install then run. Optionally specify a platform.")
print(" brrr [ip] [platform] Functionally the same as \"bir\", but \"app goes brrr\" meme variant.")
print("")
print("Options:")
print(" --help Show this commandline info")
print(" --local-sdk Use SDK specified by environment variable TACTILITY_SDK_PATH with platform subfolders matching target platforms.")
print(" --skip-build Run everything except the idf.py/CMake commands")
print(" --verbose Show extra console output")
# region Core
def download_file(url, filepath):
global verbose
if verbose:
print(f"Downloading from {url} to {filepath}")
parsed = urlparse(url)
if parsed.scheme not in ("http", "https"):
print_error(f"Unsupported URL scheme: {parsed.scheme}")
return False
request = urllib.request.Request(
url,
data=None,
headers={
"User-Agent": f"Tactility Build Tool {ttbuild_version}"
}
)
try:
with urllib.request.urlopen(request, timeout=30) as response, open(filepath, mode="wb") as file:
file.write(response.read())
return True
except OSError as error:
if verbose:
print_error(f"Failed to fetch URL {url}\n{error}")
return False
def print_warning(message):
print(f"{shell_color_orange}WARNING: {message}{shell_color_reset}")
def print_error(message):
print(f"{shell_color_red}ERROR: {message}{shell_color_reset}")
def print_status_busy(status):
sys.stdout.write(f"{status}\r")
def print_status_success(status):
# Trailing spaces are to overwrite previously written characters by a potentially shorter print_status_busy() text
print(f"{shell_color_green}{status}{shell_color_reset} ")
def print_status_error(status):
# Trailing spaces are to overwrite previously written characters by a potentially shorter print_status_busy() text
print(f"{shell_color_red}{status}{shell_color_reset} ")
def exit_with_error(message):
print_error(message)
sys.exit(1)
def get_url(ip, path):
return f"http://{ip}:{ttport}{path}"
def read_properties_file(path):
config = configparser.RawConfigParser()
config.read(path)
return config
#endregion Core
#region SDK helpers
def read_sdk_json():
json_file_path = os.path.join(ttbuild_path, "tool.json")
with open(json_file_path) as json_file:
return json.load(json_file)
def get_sdk_dir(version, platform):
global use_local_sdk, local_base_path
if use_local_sdk:
base_path = local_base_path
if base_path is None:
exit_with_error("TACTILITY_SDK_PATH environment variable is not set")
sdk_parent_dir = os.path.join(base_path, f"{version}-{platform}")
sdk_dir = os.path.join(sdk_parent_dir, "TactilitySDK")
if not os.path.isdir(sdk_dir):
exit_with_error(f"Local SDK folder not found for platform {platform}: {sdk_dir}")
return sdk_dir
else:
return os.path.join(ttbuild_path, f"{version}-{platform}", "TactilitySDK")
def validate_local_sdks(platforms, version):
if not use_local_sdk:
return
global local_base_path
base_path = local_base_path
for platform in platforms:
sdk_parent_dir = os.path.join(base_path, f"{version}-{platform}")
sdk_dir = os.path.join(sdk_parent_dir, "TactilitySDK")
if not os.path.isdir(sdk_dir):
exit_with_error(f"Local SDK folder missing for {platform}: {sdk_dir}")
def get_sdk_root_dir(version, platform):
global ttbuild_cdn
return os.path.join(ttbuild_path, f"{version}-{platform}")
def get_sdk_url(version, file):
global ttbuild_cdn
return f"{ttbuild_cdn}/sdk/{version}/{file}"
def sdk_exists(version, platform):
sdk_dir = get_sdk_dir(version, platform)
return os.path.isdir(sdk_dir)
def should_update_tool_json():
global ttbuild_cdn
json_filepath = os.path.join(ttbuild_path, "tool.json")
if os.path.exists(json_filepath):
json_modification_time = os.path.getmtime(json_filepath)
now = time.time()
global ttbuild_sdk_json_validity
minimum_seconds_difference = ttbuild_sdk_json_validity
return (now - json_modification_time) > minimum_seconds_difference
else:
return True
def update_tool_json():
global ttbuild_cdn, ttbuild_path
json_url = f"{ttbuild_cdn}/sdk/tool.json"
json_filepath = os.path.join(ttbuild_path, "tool.json")
return download_file(json_url, json_filepath)
def should_fetch_sdkconfig_files(platform_targets):
for platform in platform_targets:
sdkconfig_filename = f"sdkconfig.app.{platform}"
if not os.path.exists(os.path.join(ttbuild_path, sdkconfig_filename)):
return True
return False
def fetch_sdkconfig_files(platform_targets):
for platform in platform_targets:
sdkconfig_filename = f"sdkconfig.app.{platform}"
target_path = os.path.join(ttbuild_path, sdkconfig_filename)
if not download_file(f"{ttbuild_cdn}/{sdkconfig_filename}", target_path):
exit_with_error(f"Failed to download sdkconfig file for {platform}")
#endregion SDK helpers
#region Validation
def validate_environment():
if os.environ.get("IDF_PATH") is None:
if sys.platform == "win32":
exit_with_error("Cannot find the Espressif IDF SDK. Ensure it is installed and that it is activated via %IDF_PATH%\\export.ps1")
else:
exit_with_error("Cannot find the Espressif IDF SDK. Ensure it is installed and that it is activated via $PATH_TO_IDF_SDK/export.sh")
if not os.path.exists("manifest.properties"):
exit_with_error("manifest.properties not found")
if use_local_sdk == False and os.environ.get("TACTILITY_SDK_PATH") is not None:
print_warning("TACTILITY_SDK_PATH is set, but will be ignored by this command.")
print_warning("If you want to use it, use the '--local-sdk' parameter")
elif use_local_sdk == True and os.environ.get("TACTILITY_SDK_PATH") is None:
exit_with_error("local build was requested, but TACTILITY_SDK_PATH environment variable is not set.")
def validate_self(sdk_json):
if not "toolVersion" in sdk_json:
exit_with_error("Server returned invalid SDK data format (toolVersion not found)")
if not "toolCompatibility" in sdk_json:
exit_with_error("Server returned invalid SDK data format (toolCompatibility not found)")
if not "toolDownloadUrl" in sdk_json:
exit_with_error("Server returned invalid SDK data format (toolDownloadUrl not found)")
tool_version = sdk_json["toolVersion"]
tool_compatibility = sdk_json["toolCompatibility"]
if tool_version != ttbuild_version:
print_warning(f"New version available: {tool_version} (currently using {ttbuild_version})")
print_warning(f"Run 'tactility.py updateself' to update.")
if re.search(tool_compatibility, ttbuild_version) is None:
print_error("The tool is not compatible anymore.")
print_error("Run 'tactility.py updateself' to update.")
sys.exit(1)
#endregion Validation
#region Manifest
def read_manifest():
return read_properties_file("manifest.properties")
def validate_manifest(manifest):
# [manifest]
if not "manifest" in manifest:
exit_with_error("Invalid manifest format: [manifest] not found")
if not "version" in manifest["manifest"]:
exit_with_error("Invalid manifest format: [manifest] version not found")
# [target]
if not "target" in manifest:
exit_with_error("Invalid manifest format: [target] not found")
if not "sdk" in manifest["target"]:
exit_with_error("Invalid manifest format: [target] sdk not found")
if not "platforms" in manifest["target"]:
exit_with_error("Invalid manifest format: [target] platforms not found")
# [app]
if not "app" in manifest:
exit_with_error("Invalid manifest format: [app] not found")
if not "id" in manifest["app"]:
exit_with_error("Invalid manifest format: [app] id not found")
if not "versionName" in manifest["app"]:
exit_with_error("Invalid manifest format: [app] versionName not found")
if not "versionCode" in manifest["app"]:
exit_with_error("Invalid manifest format: [app] versionCode not found")
if not "name" in manifest["app"]:
exit_with_error("Invalid manifest format: [app] name not found")
def is_valid_manifest_platform(manifest, platform):
manifest_platforms = manifest["target"]["platforms"].split(",")
return platform in manifest_platforms
def validate_manifest_platform(manifest, platform):
if not is_valid_manifest_platform(manifest, platform):
exit_with_error(f"Platform {platform} is not available in the manifest.")
def get_manifest_target_platforms(manifest, requested_platform):
if requested_platform == "" or requested_platform is None:
return manifest["target"]["platforms"].split(",")
else:
validate_manifest_platform(manifest, requested_platform)
return [requested_platform]
#endregion Manifest
#region SDK download
def safe_extract_zip(zip_ref, target_dir):
target_dir = os.path.realpath(target_dir)
for member in zip_ref.infolist():
dest = os.path.realpath(os.path.join(target_dir, member.filename))
if not dest.startswith(target_dir + os.sep):
raise ValueError(f"Invalid zip entry: {member.filename}")
zip_ref.extractall(target_dir)
def sdk_download(version, platform):
sdk_root_dir = get_sdk_root_dir(version, platform)
os.makedirs(sdk_root_dir, exist_ok=True)
sdk_index_url = get_sdk_url(version, "index.json")
print(f"Downloading SDK version {version} for {platform}")
sdk_index_filepath = os.path.join(sdk_root_dir, "index.json")
if verbose:
print(f"Downloading {sdk_index_url} to {sdk_index_filepath}")
if not download_file(sdk_index_url, sdk_index_filepath):
# TODO: 404 check, print a more accurate error
print_error(f"Failed to download SDK version {version}. Check your internet connection and make sure this release exists.")
return False
with open(sdk_index_filepath) as sdk_index_json_file:
sdk_index_json = json.load(sdk_index_json_file)
sdk_platforms = sdk_index_json["platforms"]
if platform not in sdk_platforms:
print_error(f"Platform {platform} not found in {sdk_platforms} for version {version}")
return False
sdk_platform_file = sdk_platforms[platform]
sdk_zip_source_url = get_sdk_url(version, sdk_platform_file)
sdk_zip_target_filepath = os.path.join(sdk_root_dir, f"{version}-{platform}.zip")
if verbose:
print(f"Downloading {sdk_zip_source_url} to {sdk_zip_target_filepath}")
if not download_file(sdk_zip_source_url, sdk_zip_target_filepath):
print_error(f"Failed to download {sdk_zip_source_url} to {sdk_zip_target_filepath}")
return False
with zipfile.ZipFile(sdk_zip_target_filepath, "r") as zip_ref:
safe_extract_zip(zip_ref, os.path.join(sdk_root_dir, "TactilitySDK"))
return True
def sdk_download_all(version, platforms):
for platform in platforms:
if not sdk_exists(version, platform):
if not sdk_download(version, platform):
return False
else:
if verbose:
print(f"Using cached download for SDK version {version} and platform {platform}")
return True
#endregion SDK download
#region Building
def get_cmake_path(platform):
return os.path.join("build", f"cmake-build-{platform}")
def find_elf_file(platform):
cmake_dir = get_cmake_path(platform)
if os.path.exists(cmake_dir):
for file in os.listdir(cmake_dir):
if file.endswith(".app.elf"):
return os.path.join(cmake_dir, file)
return None
def build_all(version, platforms, skip_build):
for platform in platforms:
# First build command must be "idf.py build", otherwise it fails to execute "idf.py elf"
# We check if the ELF file exists and run the correct command
# This can lead to code caching issues, so sometimes a clean build is required
if find_elf_file(platform) is None:
if not build_first(version, platform, skip_build):
return False
else:
if not build_consecutively(version, platform, skip_build):
return False
return True
def wait_for_process(process):
buffer = []
if sys.platform != "win32":
os.set_blocking(process.stdout.fileno(), False)
while process.poll() is None:
while True:
line = process.stdout.readline()
if line:
decoded_line = line.decode("UTF-8")
if decoded_line != "":
buffer.append(decoded_line)
else:
break
else:
break
# Read any remaining output
for line in process.stdout:
decoded_line = line.decode("UTF-8")
if decoded_line:
buffer.append(decoded_line)
return buffer
# The first build must call "idf.py build" and consecutive builds must call "idf.py elf" as it finishes faster.
# The problem is that the "idf.py build" always results in an error, even though the elf file is created.
# The solution is to suppress the error if we find that the elf file was created.
def build_first(version, platform, skip_build):
sdk_dir = get_sdk_dir(version, platform)
if verbose:
print(f"Using SDK at {sdk_dir}")
os.environ["TACTILITY_SDK_PATH"] = sdk_dir
sdkconfig_path = os.path.join(ttbuild_path, f"sdkconfig.app.{platform}")
shutil.copy(sdkconfig_path, "sdkconfig")
elf_path = find_elf_file(platform)
# Remove previous elf file: re-creation of the file is used to measure if the build succeeded,
# as the actual build job will always fail due to technical issues with the elf cmake script
if elf_path is not None:
os.remove(elf_path)
if skip_build:
return True
print(f"Building first {platform} build")
cmake_path = get_cmake_path(platform)
print_status_busy(f"Building {platform} ELF")
shell_needed = sys.platform == "win32"
build_command = ["idf.py", "-B", cmake_path, "build"]
if verbose:
print(f"Running command: {' '.join(build_command)}")
with subprocess.Popen(build_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=shell_needed) as process:
build_output = wait_for_process(process)
# The return code is never expected to be 0 due to a bug in the elf cmake script, but we keep it just in case
if process.returncode == 0:
print(f"{shell_color_green}Building for {platform}{shell_color_reset}")
return True
else:
if find_elf_file(platform) is None:
for line in build_output:
print(line, end="")
print_status_error(f"Building {platform} ELF")
return False
else:
print_status_success(f"Building {platform} ELF")
return True
def build_consecutively(version, platform, skip_build):
sdk_dir = get_sdk_dir(version, platform)
if verbose:
print(f"Using SDK at {sdk_dir}")
os.environ["TACTILITY_SDK_PATH"] = sdk_dir
sdkconfig_path = os.path.join(ttbuild_path, f"sdkconfig.app.{platform}")
shutil.copy(sdkconfig_path, "sdkconfig")
if skip_build:
return True
cmake_path = get_cmake_path(platform)
print_status_busy(f"Building {platform} ELF")
shell_needed = sys.platform == "win32"
build_command = ["idf.py", "-B", cmake_path, "elf"]
if verbose:
print(f"Running command: {" ".join(build_command)}")
with subprocess.Popen(build_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=shell_needed) as process:
build_output = wait_for_process(process)
if process.returncode == 0:
print_status_success(f"Building {platform} ELF")
return True
else:
for line in build_output:
print(line, end="")
print_status_error(f"Building {platform} ELF")
return False
#endregion Building
#region Packaging
def package_intermediate_manifest(target_path):
if not os.path.isfile("manifest.properties"):
print_error("manifest.properties not found")
return False
shutil.copy("manifest.properties", os.path.join(target_path, "manifest.properties"))
return True
def package_intermediate_binaries(target_path, platforms):
elf_dir = os.path.join(target_path, "elf")
os.makedirs(elf_dir, exist_ok=True)
for platform in platforms:
elf_path = find_elf_file(platform)
if elf_path is None:
print_error(f"ELF file not found at {elf_path}")
return False
shutil.copy(elf_path, os.path.join(elf_dir, f"{platform}.elf"))
return True
def package_intermediate_assets(target_path):
if os.path.isdir("assets"):
shutil.copytree("assets", os.path.join(target_path, "assets"), dirs_exist_ok=True)
def package_intermediate(platforms):
target_path = os.path.join("build", "package-intermediate")
if os.path.isdir(target_path):
shutil.rmtree(target_path)
os.makedirs(target_path, exist_ok=True)
if not package_intermediate_manifest(target_path):
return False
if not package_intermediate_binaries(target_path, platforms):
return False
package_intermediate_assets(target_path)
return True
def package_name(platforms):
elf_path = find_elf_file(platforms[0])
elf_base_name = os.path.basename(elf_path).removesuffix(".app.elf")
return os.path.join("build", f"{elf_base_name}.app")
def package_all(platforms):
status = f"Building package with {platforms}"
print_status_busy(status)
if not package_intermediate(platforms):
print_status_error("Building package failed: missing inputs")
return False
# Create build/something.app
try:
tar_path = package_name(platforms)
tar = tarfile.open(tar_path, mode="w", format=tarfile.USTAR_FORMAT)
tar.add(os.path.join("build", "package-intermediate"), arcname="")
tar.close()
print_status_success(status)
return True
except Exception as e:
print_status_error(f"Building package failed: {e}")
return False
#endregion Packaging
def setup_environment():
global ttbuild_path
os.makedirs(ttbuild_path, exist_ok=True)
def build_action(manifest, platform_arg, skip_build):
# Environment validation
validate_environment()
platforms_to_build = get_manifest_target_platforms(manifest, platform_arg)
if use_local_sdk:
global local_base_path
local_base_path = os.environ.get("TACTILITY_SDK_PATH")
validate_local_sdks(platforms_to_build, manifest["target"]["sdk"])
if should_fetch_sdkconfig_files(platforms_to_build):
fetch_sdkconfig_files(platforms_to_build)
if not use_local_sdk:
sdk_json = read_sdk_json()
validate_self(sdk_json)
# Build
sdk_version = manifest["target"]["sdk"]
if not use_local_sdk:
if not sdk_download_all(sdk_version, platforms_to_build):
exit_with_error("Failed to download one or more SDKs")
if not build_all(sdk_version, platforms_to_build, skip_build): # Environment validation
return False
if not skip_build:
if not package_all(platforms_to_build):
return False
return True
def clean_action():
if os.path.exists("build"):
print_status_busy("Removing build/")
shutil.rmtree("build")
print_status_success("Removed build/")
else:
print("Nothing to clean")
def clear_cache_action():
if os.path.exists(ttbuild_path):
print_status_busy(f"Removing {ttbuild_path}/")
shutil.rmtree(ttbuild_path)
print_status_success(f"Removed {ttbuild_path}/")
else:
print("Nothing to clear")
def update_self_action():
sdk_json = read_sdk_json()
tool_download_url = sdk_json["toolDownloadUrl"]
if download_file(tool_download_url, "tactility.py"):
print("Updated")
else:
exit_with_error("Update failed")
def get_device_info(ip):
print_status_busy(f"Requesting device info")
url = get_url(ip, "/info")
try:
response = requests.get(url, timeout=http_timeout_seconds)
if response.status_code != 200:
print_error("Run failed")
else:
print_status_success(f"Received device info:")
print(response.json())
except requests.RequestException as e:
print_status_error(f"Device info request failed: {e}")
def run_action(manifest, ip):
app_id = manifest["app"]["id"]
print_status_busy("Running")
url = get_url(ip, "/app/run")
params = {'id': app_id}
try:
response = requests.post(url, params=params, timeout=http_timeout_seconds)
if response.status_code != 200:
print_error("Run failed")
else:
print_status_success("Running")
except requests.RequestException as e:
print_status_error(f"Running request failed: {e}")
def install_action(ip, platforms):
print_status_busy("Installing")
for platform in platforms:
elf_path = find_elf_file(platform)
if elf_path is None:
print_status_error(f"ELF file not built for {platform}")
return False
package_path = package_name(platforms)
# print(f"Installing {package_path} to {ip}")
url = get_url(ip, "/app/install")
try:
# Prepare multipart form data
with open(package_path, 'rb') as file:
files = {
'elf': file
}
response = requests.put(url, files=files, timeout=http_timeout_seconds)
if response.status_code != 200:
print_status_error("Install failed")
return False
else:
print_status_success("Installing")
return True
except requests.RequestException as e:
print_status_error(f"Install request failed: {e}")
return False
except IOError as e:
print_status_error(f"Install file error: {e}")
return False
def uninstall_action(manifest, ip):
app_id = manifest["app"]["id"]
print_status_busy("Uninstalling")
url = get_url(ip, "/app/uninstall")
params = {'id': app_id}
try:
response = requests.put(url, params=params, timeout=http_timeout_seconds)
if response.status_code != 200:
print_status_error("Server responded that uninstall failed")
else:
print_status_success("Uninstalled")
except requests.RequestException as e:
print_status_error(f"Uninstall request failed: {e}")
#region Main
if __name__ == "__main__":
print(f"Tactility Build System v{ttbuild_version}")
if "--help" in sys.argv:
print_help()
sys.exit()
# Argument validation
if len(sys.argv) == 1:
print_help()
sys.exit(1)
if "--verbose" in sys.argv:
verbose = True
sys.argv.remove("--verbose")
skip_build = False
if "--skip-build" in sys.argv:
skip_build = True
sys.argv.remove("--skip-build")
if "--local-sdk" in sys.argv:
use_local_sdk = True
sys.argv.remove("--local-sdk")
action_arg = sys.argv[1]
# Environment setup
setup_environment()
if not os.path.isfile("manifest.properties"):
exit_with_error("manifest.properties not found")
manifest = read_manifest()
validate_manifest(manifest)
all_platform_targets = manifest["target"]["platforms"].split(",")
# Update SDK cache (tool.json)
if not use_local_sdk and should_update_tool_json() and not update_tool_json():
exit_with_error("Failed to retrieve SDK info")
# Actions
if action_arg == "build":
if len(sys.argv) < 2:
print_help()
exit_with_error("Commandline parameter missing")
platform = None
if len(sys.argv) > 2:
platform = sys.argv[2]
if not build_action(manifest, platform, skip_build):
sys.exit(1)
elif action_arg == "clean":
clean_action()
elif action_arg == "clearcache":
clear_cache_action()
elif action_arg == "updateself":
update_self_action()
elif action_arg == "run":
if len(sys.argv) < 3:
print_help()
exit_with_error("Commandline parameter missing")
run_action(manifest, sys.argv[2])
elif action_arg == "install":
if len(sys.argv) < 3:
print_help()
exit_with_error("Commandline parameter missing")
platform = None
platforms_to_install = all_platform_targets
if len(sys.argv) >= 4:
platform = sys.argv[3]
platforms_to_install = [platform]
install_action(sys.argv[2], platforms_to_install)
elif action_arg == "uninstall":
if len(sys.argv) < 3:
print_help()
exit_with_error("Commandline parameter missing")
uninstall_action(manifest, sys.argv[2])
elif action_arg == "bir" or action_arg == "brrr":
if len(sys.argv) < 3:
print_help()
exit_with_error("Commandline parameter missing")
platform = None
platforms_to_install = all_platform_targets
if len(sys.argv) >= 4:
platform = sys.argv[3]
platforms_to_install = [platform]
if build_action(manifest, platform, skip_build):
if install_action(sys.argv[2], platforms_to_install):
run_action(manifest, sys.argv[2])
else:
print_help()
exit_with_error("Unknown commandline parameter")
#endregion Main