diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d3846a243..f330a3c00 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,8 +46,23 @@ jobs: persist-credentials: false - name: "Build Tools" uses: ./.github/actions/build-tools + # Always runs, so the jobs that depend on it aren't skipped for pushes + CheckTtVersion: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + if: github.event_name == 'pull_request' + with: + persist-credentials: false + fetch-depth: 0 + - name: "Check tt version" + if: github.event_name == 'pull_request' + env: + BASE_REF: ${{ github.base_ref }} + run: python3 Buildscripts/check-tt-version.py "origin/$BASE_REF" BuildTt: runs-on: ubuntu-latest + needs: [ CheckTtVersion ] steps: - uses: actions/checkout@v6 with: diff --git a/Buildscripts/CDN/tactility.py b/Buildscripts/CDN/tactility.py index 849591266..eaf485302 100644 --- a/Buildscripts/CDN/tactility.py +++ b/Buildscripts/CDN/tactility.py @@ -55,7 +55,7 @@ if __name__ == "__main__": if not install_tt(directory): sys.exit(1) print(f"Installed tt in {directory}") - command = "tt" if os.name == "nt" else "./tt" + command = "py tt.py" if os.name == "nt" else "./tt" print_warning(f"tactility.py was replaced by tt: use tt from now on, e.g. '{command} app build'") print_warning(f"Run '{command} help' for more info") sys.exit(1) diff --git a/Buildscripts/TactilitySDK/Tools/_bootstrap.py b/Buildscripts/TactilitySDK/Tools/_bootstrap.py index c0d19e5f7..5fa6f9919 100755 --- a/Buildscripts/TactilitySDK/Tools/_bootstrap.py +++ b/Buildscripts/TactilitySDK/Tools/_bootstrap.py @@ -56,6 +56,8 @@ def check_cdn(state, sdk_version): index = fetch_json(f"{CDN_URL}/sdk/{sdk_version}/index.json") if isinstance(index, dict): state.setdefault("sdk", {})[sdk_version] = index.get("gitCommit") + tools = index.get("tools") + state.setdefault("sdk_tools", {})[sdk_version] = tools.get("sha256") if isinstance(tools, dict) else None write_state(state) @@ -75,6 +77,14 @@ def installed_commits(version_directory): return commits +def installed_tools_hash(version_directory): + try: + with open(os.path.join(version_directory, "tools.sha256")) as file: + return file.read().strip() + except OSError: + return None + + def run(): tt_version = os.environ.get("TT_VERSION") if tt_version is None: @@ -91,8 +101,14 @@ def run(): latest = state.get("version") if isinstance(latest, str) and parse_version(latest) > parse_version(tt_version): print(f"tt: version {latest} is available (currently {tt_version}): run 'tt update-self'", file=sys.stderr) - latest_commit = state.get("sdk", {}).get(sdk_version) if sdk_version is not None else None - if latest_commit and sdk_path is not None and any(commit != latest_commit for commit in installed_commits(os.path.join(sdk_path, "sdk", sdk_version))): + if sdk_version is None or sdk_path is None: + return + version_directory = os.path.join(sdk_path, "sdk", sdk_version) + latest_commit = state.get("sdk", {}).get(sdk_version) + latest_tools = state.get("sdk_tools", {}).get(sdk_version) + has_newer_platforms = latest_commit and any(commit != latest_commit for commit in installed_commits(version_directory)) + has_newer_tools = latest_tools and installed_tools_hash(version_directory) != latest_tools + if has_newer_platforms or has_newer_tools: print(f"tt: a newer build of SDK {sdk_version} is available: run 'tt update-sdk'", file=sys.stderr) diff --git a/Buildscripts/TactilitySDK/Tools/update-sdk.py b/Buildscripts/TactilitySDK/Tools/update-sdk.py index 693016210..e32c1c868 100755 --- a/Buildscripts/TactilitySDK/Tools/update-sdk.py +++ b/Buildscripts/TactilitySDK/Tools/update-sdk.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 # License: Apache License v2 -"""Downloads the installed platforms of the selected SDK again when the CDN has a newer build of it. Run it as "tt update-sdk".""" +"""Downloads the tools and installed platforms of the selected SDK again when the CDN has a newer build of it. Run it as "tt update-sdk".""" +import hashlib import io import json import os @@ -43,9 +44,17 @@ def installed_platforms(version_directory): ) -def install_platform(platform_directory, archive_data, commit): - """Extracts next to the installed SDK and then replaces it, refusing entries outside of it""" - staging_directory = os.path.join(platform_directory, "TactilitySDK.new") +def read_installed_tools_hash(version_directory): + try: + with open(os.path.join(version_directory, "tools.sha256")) as file: + return file.read().strip() + except OSError: + return None + + +def extract(archive_data, target_directory): + """Extracts next to target_directory and then replaces it, refusing entries outside of it""" + staging_directory = f"{target_directory}.new" shutil.rmtree(staging_directory, ignore_errors=True) os.makedirs(staging_directory) root = os.path.realpath(staging_directory) @@ -54,18 +63,27 @@ def install_platform(platform_directory, archive_data, commit): for member in archive.infolist(): destination = os.path.realpath(os.path.join(root, member.filename)) if not destination.startswith(root + os.sep): - fail(f"invalid entry in the SDK archive: {member.filename}") + fail(f"invalid entry in the archive: {member.filename}") archive.extractall(root) except zipfile.BadZipFile: shutil.rmtree(staging_directory, ignore_errors=True) - fail("the downloaded SDK archive is invalid") - sdk_directory = os.path.join(platform_directory, "TactilitySDK") - shutil.rmtree(sdk_directory, ignore_errors=True) - os.replace(staging_directory, sdk_directory) + fail("the downloaded archive is invalid") + shutil.rmtree(target_directory, ignore_errors=True) + os.replace(staging_directory, target_directory) + + +def install_platform(platform_directory, archive_data, commit): + extract(archive_data, os.path.join(platform_directory, "TactilitySDK")) with open(os.path.join(platform_directory, "commit"), "w") as file: file.write(commit) +def install_tools(version_directory, archive_data, sha256): + extract(archive_data, os.path.join(version_directory, "tools")) + with open(os.path.join(version_directory, "tools.sha256"), "w") as file: + file.write(sha256) + + def main(): version = os.environ.get("TACTILITY_SDK_VERSION") sdk_path = os.environ.get("TACTILITY_SDK_PATH") @@ -82,6 +100,14 @@ def main(): commit = index.get("gitCommit", "") available = index.get("platforms", {}) updated = False + tools = index.get("tools") + if tools is not None and read_installed_tools_hash(version_directory) != tools.get("sha256"): + print(f"Updating the tools of SDK {version}") + archive_data = fetch(f"{CDN_URL}/sdk/{version}/{tools['file']}") + if hashlib.sha256(archive_data).hexdigest() != tools["sha256"]: + fail("the downloaded tools don't match their checksum") + install_tools(version_directory, archive_data, tools["sha256"]) + updated = True for platform in installed_platforms(version_directory): platform_directory = os.path.join(version_directory, f"platform-{platform}") if read_commit(platform_directory) == commit or platform not in available: diff --git a/Buildscripts/TactilitySDK/Tools/update-self.py b/Buildscripts/TactilitySDK/Tools/update-self.py index b9a6d2ad5..014e3173a 100755 --- a/Buildscripts/TactilitySDK/Tools/update-self.py +++ b/Buildscripts/TactilitySDK/Tools/update-self.py @@ -11,6 +11,8 @@ import zipfile CDN_URL = os.environ.get("TT_CDN", "https://cdn.tactilityproject.org") FILE_NAME_PATTERN = re.compile(r"^[A-Za-z0-9_-][A-Za-z0-9_.-]*$") +# Files of older versions of tt: tt.cmd made cmd.exe parse the arguments a second time, tt.ps1 replaces it +OBSOLETE_FILE_NAMES = ["tt.cmd"] def fail(message): @@ -44,16 +46,15 @@ def main(): fail("the downloaded tt.py is invalid") for file_name, content in contents.items(): path = os.path.join(directory, file_name) - # A running tt.cmd is read while it executes, so an unchanged file is left alone - if os.path.isfile(path): - with open(path, "rb") as file: - if file.read() == content: - continue temporary_path = f"{path}.new" with open(temporary_path, "wb") as file: file.write(content) os.chmod(temporary_path, 0o755) os.replace(temporary_path, path) + for file_name in OBSOLETE_FILE_NAMES: + path = os.path.join(directory, file_name) + if file_name not in contents and os.path.isfile(path): + os.remove(path) print("tt was updated") diff --git a/Buildscripts/check-tt-version.py b/Buildscripts/check-tt-version.py new file mode 100755 index 000000000..233d3385b --- /dev/null +++ b/Buildscripts/check-tt-version.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Fails when the files of tt changed since a base revision, but TT_VERSION in tt.py wasn't increased. +Usage: check-tt-version.py , e.g. origin/main""" + +import importlib.util +import os +import re +import subprocess +import sys + +ROOT_PATH = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) +VERSION_PATTERN = re.compile(r'^TT_VERSION = "([^"]+)"', re.MULTILINE) + + +def load_file_names(): + """The files that make up tt, as release-tt.py packages them""" + spec = importlib.util.spec_from_file_location("release_tt", os.path.join(ROOT_PATH, "Buildscripts", "release-tt.py")) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.FILE_NAMES + + +def git(*arguments): + return subprocess.run(["git", *arguments], cwd=ROOT_PATH, capture_output=True, text=True) + + +def read_version(source): + match = VERSION_PATTERN.search(source) + return match.group(1) if match else None + + +def parse_version(version): + return tuple(int(part) for part in re.findall(r"\d+", version)[:3]) + + +def main(): + if len(sys.argv) != 2: + print("Usage: check-tt-version.py ", file=sys.stderr) + sys.exit(2) + base = sys.argv[1] + file_names = load_file_names() + diff = git("diff", "--name-only", f"{base}...HEAD", "--", *file_names) + if diff.returncode != 0: + print(f"git diff failed: {diff.stderr.strip()}", file=sys.stderr) + sys.exit(2) + changed = diff.stdout.split() + if not changed: + print("tt is unchanged") + return + base_source = git("show", f"{base}:tt.py") + if base_source.returncode != 0: + print("tt is new") + return + with open(os.path.join(ROOT_PATH, "tt.py")) as file: + current_version = read_version(file.read()) + base_version = read_version(base_source.stdout) + if current_version is None or base_version is None: + print("TT_VERSION not found in tt.py", file=sys.stderr) + sys.exit(1) + if parse_version(current_version) <= parse_version(base_version): + print(f"{', '.join(changed)} changed, but TT_VERSION is still {current_version}: increase TT_VERSION in tt.py", file=sys.stderr) + sys.exit(1) + print(f"tt changed and its version increased from {base_version} to {current_version}") + + +if __name__ == "__main__": + main() diff --git a/Buildscripts/release-tt.py b/Buildscripts/release-tt.py index 435f35fba..ccc3dc9a0 100755 --- a/Buildscripts/release-tt.py +++ b/Buildscripts/release-tt.py @@ -10,8 +10,8 @@ import zipfile ROOT_PATH = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) LAUNCHER_PATH = os.path.join(ROOT_PATH, "tt.py") -# tt.py and the wrappers that run it: tt (Linux, macOS) and tt.cmd (Windows) -FILE_NAMES = ["tt.py", "tt", "tt.cmd"] +# tt.py and the wrappers that run it: tt (Linux, macOS) and tt.ps1 (Windows PowerShell) +FILE_NAMES = ["tt.py", "tt", "tt.ps1"] CDN_OUTPUT_PATH = os.path.join(ROOT_PATH, "release", "tt") diff --git a/Tests/Python/tt_test.py b/Tests/Python/tt_test.py index 22b84359e..674e2c38f 100644 --- a/Tests/Python/tt_test.py +++ b/Tests/Python/tt_test.py @@ -160,22 +160,30 @@ class TtTest(unittest.TestCase): self.assertIn("echo", result.stdout) self.assertIn(VERSION, result.stdout) - def test_changed_tools_are_downloaded_again(self): + def test_installed_tools_and_platforms_are_only_replaced_by_update_sdk(self): self.write_manifest(self.work) self.assertEqual(self.run_tt("echo").returncode, 3) - self.publish_sdk({"echo.py": "import sys\nsys.exit(4)\n"}) - self.assertEqual(self.run_tt("echo").returncode, 3) + tools = self.default_tools() | {"echo.py": "import sys\nimport _bootstrap\n_bootstrap.run()\nsys.exit(4)\n"} + self.publish_sdk(tools, commit="commit2", sdk_content="# sdk 2") self.expire_index() + self.expire_update_check() + result = self.run_tt("echo") + self.assertEqual(result.returncode, 3) + self.assertEqual(self.read_sdk_file(), "# sdk") + self.assertIn("run 'tt update-sdk'", result.stderr) + result = self.run_tt("update-sdk") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("Updating the tools", result.stdout) self.assertEqual(self.run_tt("echo").returncode, 4) + self.assertEqual(self.read_sdk_file(), "# sdk 2") - def test_failed_download_keeps_the_cache(self): + def test_installed_sdk_works_offline(self): self.write_manifest(self.work) self.assertEqual(self.run_tt("echo").returncode, 3) shutil.rmtree(self.cdn) self.expire_index() result = self.run_tt("echo") self.assertEqual(result.returncode, 3, result.stderr) - self.assertIn("cached", result.stderr) def test_tools_with_a_wrong_checksum_are_refused(self): self.write_manifest(self.work) @@ -225,16 +233,19 @@ class TtTest(unittest.TestCase): def test_update_replaces_itself_and_installs_wrappers(self): tt_copy = os.path.join(self.root, "tt.py") shutil.copy(TT_PATH, tt_copy) - self.publish_tt({"tt.py": "# updated\n", "tt": "#!/bin/sh\n", "tt.cmd": "@echo off\r\n"}) + with open(os.path.join(self.root, "tt.cmd"), "w") as file: + file.write("@echo off\r\n") + self.publish_tt({"tt.py": "# updated\n", "tt": "#!/bin/sh\n", "tt.ps1": "exit 0\r\n"}) self.write_manifest(self.work) result = self.run_tt("update-self", tt_path=tt_copy) self.assertEqual(result.returncode, 0, result.stderr) with open(tt_copy) as file: self.assertEqual(file.read(), "# updated\n") self.assertTrue(os.access(os.path.join(self.root, "tt"), os.X_OK)) - self.assertTrue(os.access(os.path.join(self.root, "tt.cmd"), os.X_OK)) + self.assertTrue(os.access(os.path.join(self.root, "tt.ps1"), os.X_OK)) self.assertTrue(os.access(tt_copy, os.X_OK)) - self.assertEqual(sorted(os.listdir(self.root)), ["cdn", "home", "tt", "tt.cmd", "tt.py", "work"]) + # The obsolete tt.cmd is removed + self.assertEqual(sorted(os.listdir(self.root)), ["cdn", "home", "tt", "tt.ps1", "tt.py", "work"]) def test_update_rejects_invalid_python(self): tt_copy = os.path.join(self.root, "tt.py") diff --git a/tt.cmd b/tt.cmd deleted file mode 100755 index fc685aaf4..000000000 --- a/tt.cmd +++ /dev/null @@ -1,9 +0,0 @@ -@echo off -rem Runs tt.py from the directory of this script, on Windows -where py >nul 2>nul -if %ERRORLEVEL% equ 0 ( - py -3 "%~dp0tt.py" %* -) else ( - python "%~dp0tt.py" %* -) -exit /b %ERRORLEVEL% diff --git a/tt.ps1 b/tt.ps1 new file mode 100755 index 000000000..4cc5537ba --- /dev/null +++ b/tt.ps1 @@ -0,0 +1,9 @@ +# Runs tt.py from the folder of this script, on Windows. +# PowerShell passes the arguments to Python as they are, without parsing them again. +$script = Join-Path $PSScriptRoot "tt.py" +if (Get-Command py -ErrorAction SilentlyContinue) { + & py -3 $script @args +} else { + & python $script @args +} +exit $LASTEXITCODE diff --git a/tt.py b/tt.py index 667c46c0a..f05ce0820 100755 --- a/tt.py +++ b/tt.py @@ -5,7 +5,7 @@ "tt [arguments]" runs .py from the tools of that SDK version. When TACTILITY_SDK_PATH is set, its SDK and tools are used instead of downloading them. Both ~/.tactility and TACTILITY_SDK_PATH hold sdk//tools/ and sdk//platform-/TactilitySDK/. -The tt (Linux, macOS) and tt.cmd (Windows) wrappers next to this script run it. +The tt (Linux, macOS) and tt.ps1 (Windows PowerShell) wrappers next to this script run it. The SDK version is read from ./manifest.properties, or else from the first Apps/*/manifest.properties or apps/*/manifest.properties. """ @@ -20,7 +20,7 @@ import time import urllib.request import zipfile -TT_VERSION = "1.0.0" +TT_VERSION = "1.1.0" CDN_URL = os.environ.get("TT_CDN", "https://cdn.tactilityproject.org") HOME_DIRECTORY = os.path.join(os.path.expanduser("~"), ".tactility") INDEX_VALIDITY_SECONDS = 3600 @@ -143,16 +143,9 @@ def install_tools(version, version_directory, index): fail(f"SDK {version} has no tools") tools_directory = os.path.join(version_directory, "tools") hash_path = os.path.join(version_directory, "tools.sha256") - if os.path.isdir(tools_directory) and os.path.isfile(hash_path): - with open(hash_path) as file: - if file.read().strip() == tools["sha256"]: - return print(f"Downloading the tools of SDK {version}", file=sys.stderr) zip_path = os.path.join(version_directory, "tools.zip") if not download(sdk_url(version, tools["file"]), zip_path): - if os.path.isdir(tools_directory): - warn(f"using the cached tools of SDK {version}") - return fail(f"failed to download the tools of SDK {version}") if file_sha256(zip_path) != tools["sha256"]: os.remove(zip_path) @@ -167,8 +160,6 @@ def install_platforms(version, version_directory, index, platforms): available = index.get("platforms", {}) for platform in platforms: platform_directory = os.path.join(version_directory, f"platform-{platform}") - if os.path.isdir(os.path.join(platform_directory, "TactilitySDK")): - continue if platform not in available: fail(f"platform {platform} is not available in SDK {version}") print(f"Downloading SDK {version} for {platform}", file=sys.stderr) @@ -183,11 +174,17 @@ def install_platforms(version, version_directory, index, platforms): def prepare_sdk(version, platforms): + """Installs what's missing of the SDK version. Installed tools and platforms are only replaced by "tt update-sdk", so they stay from the same build.""" version_directory = version_path(HOME_DIRECTORY, version) + has_tools = os.path.isdir(os.path.join(version_directory, "tools")) + missing_platforms = [platform for platform in platforms if not os.path.isdir(os.path.join(version_directory, f"platform-{platform}", "TactilitySDK"))] + if has_tools and not missing_platforms: + return version_directory os.makedirs(version_directory, exist_ok=True) index = load_index(version, version_directory) - install_tools(version, version_directory, index) - install_platforms(version, version_directory, index, platforms) + if not has_tools: + install_tools(version, version_directory, index) + install_platforms(version, version_directory, index, missing_platforms) return version_directory