Build tool improvements

.github/workflows/build.yml

@@ -46,8 +46,23 @@ jobs:
           persist-credentials: false
       - name: "Build Tools"
         uses: ./.github/actions/build-tools
+  # Always runs, so the jobs that depend on it aren't skipped for pushes
+  CheckTtVersion:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        if: github.event_name == 'pull_request'
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+      - name: "Check tt version"
+        if: github.event_name == 'pull_request'
+        env:
+          BASE_REF: ${{ github.base_ref }}
+        run: python3 Buildscripts/check-tt-version.py "origin/$BASE_REF"
   BuildTt:
     runs-on: ubuntu-latest
+    needs: [ CheckTtVersion ]
     steps:
       - uses: actions/checkout@v6
         with:

Buildscripts/CDN/tactility.py

@@ -55,7 +55,7 @@ if __name__ == "__main__":
         if not install_tt(directory):
             sys.exit(1)
         print(f"Installed tt in {directory}")
-    command = "tt" if os.name == "nt" else "./tt"
+    command = "py tt.py" if os.name == "nt" else "./tt"
     print_warning(f"tactility.py was replaced by tt: use tt from now on, e.g. '{command} app build'")
     print_warning(f"Run '{command} help' for more info")
     sys.exit(1)

Buildscripts/TactilitySDK/Tools/_bootstrap.py

@@ -56,6 +56,8 @@ def check_cdn(state, sdk_version):
         index = fetch_json(f"{CDN_URL}/sdk/{sdk_version}/index.json")
         if isinstance(index, dict):
             state.setdefault("sdk", {})[sdk_version] = index.get("gitCommit")
+            tools = index.get("tools")
+            state.setdefault("sdk_tools", {})[sdk_version] = tools.get("sha256") if isinstance(tools, dict) else None
     write_state(state)
 
 
@@ -75,6 +77,14 @@ def installed_commits(version_directory):
     return commits
 
 
+def installed_tools_hash(version_directory):
+    try:
+        with open(os.path.join(version_directory, "tools.sha256")) as file:
+            return file.read().strip()
+    except OSError:
+        return None
+
+
 def run():
     tt_version = os.environ.get("TT_VERSION")
     if tt_version is None:
@@ -91,8 +101,14 @@ def run():
     latest = state.get("version")
     if isinstance(latest, str) and parse_version(latest) > parse_version(tt_version):
         print(f"tt: version {latest} is available (currently {tt_version}): run 'tt update-self'", file=sys.stderr)
-    latest_commit = state.get("sdk", {}).get(sdk_version) if sdk_version is not None else None
-    if latest_commit and sdk_path is not None and any(commit != latest_commit for commit in installed_commits(os.path.join(sdk_path, "sdk", sdk_version))):
+    if sdk_version is None or sdk_path is None:
+        return
+    version_directory = os.path.join(sdk_path, "sdk", sdk_version)
+    latest_commit = state.get("sdk", {}).get(sdk_version)
+    latest_tools = state.get("sdk_tools", {}).get(sdk_version)
+    has_newer_platforms = latest_commit and any(commit != latest_commit for commit in installed_commits(version_directory))
+    has_newer_tools = latest_tools and installed_tools_hash(version_directory) != latest_tools
+    if has_newer_platforms or has_newer_tools:
         print(f"tt: a newer build of SDK {sdk_version} is available: run 'tt update-sdk'", file=sys.stderr)
 
 

Buildscripts/TactilitySDK/Tools/update-sdk.py

@@ -1,7 +1,8 @@
 #!/usr/bin/env python3
 # License: Apache License v2
-"""Downloads the installed platforms of the selected SDK again when the CDN has a newer build of it. Run it as "tt update-sdk"."""
+"""Downloads the tools and installed platforms of the selected SDK again when the CDN has a newer build of it. Run it as "tt update-sdk"."""
 
+import hashlib
 import io
 import json
 import os
@@ -43,9 +44,17 @@ def installed_platforms(version_directory):
     )
 
 
-def install_platform(platform_directory, archive_data, commit):
-    """Extracts next to the installed SDK and then replaces it, refusing entries outside of it"""
-    staging_directory = os.path.join(platform_directory, "TactilitySDK.new")
+def read_installed_tools_hash(version_directory):
+    try:
+        with open(os.path.join(version_directory, "tools.sha256")) as file:
+            return file.read().strip()
+    except OSError:
+        return None
+
+
+def extract(archive_data, target_directory):
+    """Extracts next to target_directory and then replaces it, refusing entries outside of it"""
+    staging_directory = f"{target_directory}.new"
     shutil.rmtree(staging_directory, ignore_errors=True)
     os.makedirs(staging_directory)
     root = os.path.realpath(staging_directory)
@@ -54,18 +63,27 @@ def install_platform(platform_directory, archive_data, commit):
             for member in archive.infolist():
                 destination = os.path.realpath(os.path.join(root, member.filename))
                 if not destination.startswith(root + os.sep):
-                    fail(f"invalid entry in the SDK archive: {member.filename}")
+                    fail(f"invalid entry in the archive: {member.filename}")
             archive.extractall(root)
     except zipfile.BadZipFile:
         shutil.rmtree(staging_directory, ignore_errors=True)
-        fail("the downloaded SDK archive is invalid")
-    sdk_directory = os.path.join(platform_directory, "TactilitySDK")
-    shutil.rmtree(sdk_directory, ignore_errors=True)
-    os.replace(staging_directory, sdk_directory)
+        fail("the downloaded archive is invalid")
+    shutil.rmtree(target_directory, ignore_errors=True)
+    os.replace(staging_directory, target_directory)
+
+
+def install_platform(platform_directory, archive_data, commit):
+    extract(archive_data, os.path.join(platform_directory, "TactilitySDK"))
     with open(os.path.join(platform_directory, "commit"), "w") as file:
         file.write(commit)
 
 
+def install_tools(version_directory, archive_data, sha256):
+    extract(archive_data, os.path.join(version_directory, "tools"))
+    with open(os.path.join(version_directory, "tools.sha256"), "w") as file:
+        file.write(sha256)
+
+
 def main():
     version = os.environ.get("TACTILITY_SDK_VERSION")
     sdk_path = os.environ.get("TACTILITY_SDK_PATH")
@@ -82,6 +100,14 @@ def main():
     commit = index.get("gitCommit", "")
     available = index.get("platforms", {})
     updated = False
+    tools = index.get("tools")
+    if tools is not None and read_installed_tools_hash(version_directory) != tools.get("sha256"):
+        print(f"Updating the tools of SDK {version}")
+        archive_data = fetch(f"{CDN_URL}/sdk/{version}/{tools['file']}")
+        if hashlib.sha256(archive_data).hexdigest() != tools["sha256"]:
+            fail("the downloaded tools don't match their checksum")
+        install_tools(version_directory, archive_data, tools["sha256"])
+        updated = True
     for platform in installed_platforms(version_directory):
         platform_directory = os.path.join(version_directory, f"platform-{platform}")
         if read_commit(platform_directory) == commit or platform not in available:

Buildscripts/TactilitySDK/Tools/update-self.py

@@ -11,6 +11,8 @@ import zipfile
 
 CDN_URL = os.environ.get("TT_CDN", "https://cdn.tactilityproject.org")
 FILE_NAME_PATTERN = re.compile(r"^[A-Za-z0-9_-][A-Za-z0-9_.-]*$")
+# Files of older versions of tt: tt.cmd made cmd.exe parse the arguments a second time, tt.ps1 replaces it
+OBSOLETE_FILE_NAMES = ["tt.cmd"]
 
 
 def fail(message):
@@ -44,16 +46,15 @@ def main():
         fail("the downloaded tt.py is invalid")
     for file_name, content in contents.items():
         path = os.path.join(directory, file_name)
-        # A running tt.cmd is read while it executes, so an unchanged file is left alone
-        if os.path.isfile(path):
-            with open(path, "rb") as file:
-                if file.read() == content:
-                    continue
         temporary_path = f"{path}.new"
         with open(temporary_path, "wb") as file:
             file.write(content)
         os.chmod(temporary_path, 0o755)
         os.replace(temporary_path, path)
+    for file_name in OBSOLETE_FILE_NAMES:
+        path = os.path.join(directory, file_name)
+        if file_name not in contents and os.path.isfile(path):
+            os.remove(path)
     print("tt was updated")
 
 

Buildscripts/check-tt-version.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""Fails when the files of tt changed since a base revision, but TT_VERSION in tt.py wasn't increased.
+Usage: check-tt-version.py <base revision>, e.g. origin/main"""
+
+import importlib.util
+import os
+import re
+import subprocess
+import sys
+
+ROOT_PATH = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
+VERSION_PATTERN = re.compile(r'^TT_VERSION = "([^"]+)"', re.MULTILINE)
+
+
+def load_file_names():
+    """The files that make up tt, as release-tt.py packages them"""
+    spec = importlib.util.spec_from_file_location("release_tt", os.path.join(ROOT_PATH, "Buildscripts", "release-tt.py"))
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module.FILE_NAMES
+
+
+def git(*arguments):
+    return subprocess.run(["git", *arguments], cwd=ROOT_PATH, capture_output=True, text=True)
+
+
+def read_version(source):
+    match = VERSION_PATTERN.search(source)
+    return match.group(1) if match else None
+
+
+def parse_version(version):
+    return tuple(int(part) for part in re.findall(r"\d+", version)[:3])
+
+
+def main():
+    if len(sys.argv) != 2:
+        print("Usage: check-tt-version.py <base revision>", file=sys.stderr)
+        sys.exit(2)
+    base = sys.argv[1]
+    file_names = load_file_names()
+    diff = git("diff", "--name-only", f"{base}...HEAD", "--", *file_names)
+    if diff.returncode != 0:
+        print(f"git diff failed: {diff.stderr.strip()}", file=sys.stderr)
+        sys.exit(2)
+    changed = diff.stdout.split()
+    if not changed:
+        print("tt is unchanged")
+        return
+    base_source = git("show", f"{base}:tt.py")
+    if base_source.returncode != 0:
+        print("tt is new")
+        return
+    with open(os.path.join(ROOT_PATH, "tt.py")) as file:
+        current_version = read_version(file.read())
+    base_version = read_version(base_source.stdout)
+    if current_version is None or base_version is None:
+        print("TT_VERSION not found in tt.py", file=sys.stderr)
+        sys.exit(1)
+    if parse_version(current_version) <= parse_version(base_version):
+        print(f"{', '.join(changed)} changed, but TT_VERSION is still {current_version}: increase TT_VERSION in tt.py", file=sys.stderr)
+        sys.exit(1)
+    print(f"tt changed and its version increased from {base_version} to {current_version}")
+
+
+if __name__ == "__main__":
+    main()

Buildscripts/release-tt.py

@@ -10,8 +10,8 @@ import zipfile
 
 ROOT_PATH = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
 LAUNCHER_PATH = os.path.join(ROOT_PATH, "tt.py")
-# tt.py and the wrappers that run it: tt (Linux, macOS) and tt.cmd (Windows)
-FILE_NAMES = ["tt.py", "tt", "tt.cmd"]
+# tt.py and the wrappers that run it: tt (Linux, macOS) and tt.ps1 (Windows PowerShell)
+FILE_NAMES = ["tt.py", "tt", "tt.ps1"]
 CDN_OUTPUT_PATH = os.path.join(ROOT_PATH, "release", "tt")
 
 

Tests/Python/tt_test.py

@@ -160,22 +160,30 @@ class TtTest(unittest.TestCase):
         self.assertIn("echo", result.stdout)
         self.assertIn(VERSION, result.stdout)
 
-    def test_changed_tools_are_downloaded_again(self):
+    def test_installed_tools_and_platforms_are_only_replaced_by_update_sdk(self):
         self.write_manifest(self.work)
         self.assertEqual(self.run_tt("echo").returncode, 3)
-        self.publish_sdk({"echo.py": "import sys\nsys.exit(4)\n"})
-        self.assertEqual(self.run_tt("echo").returncode, 3)
+        tools = self.default_tools() | {"echo.py": "import sys\nimport _bootstrap\n_bootstrap.run()\nsys.exit(4)\n"}
+        self.publish_sdk(tools, commit="commit2", sdk_content="# sdk 2")
         self.expire_index()
+        self.expire_update_check()
+        result = self.run_tt("echo")
+        self.assertEqual(result.returncode, 3)
+        self.assertEqual(self.read_sdk_file(), "# sdk")
+        self.assertIn("run 'tt update-sdk'", result.stderr)
+        result = self.run_tt("update-sdk")
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertIn("Updating the tools", result.stdout)
         self.assertEqual(self.run_tt("echo").returncode, 4)
+        self.assertEqual(self.read_sdk_file(), "# sdk 2")
 
-    def test_failed_download_keeps_the_cache(self):
+    def test_installed_sdk_works_offline(self):
         self.write_manifest(self.work)
         self.assertEqual(self.run_tt("echo").returncode, 3)
         shutil.rmtree(self.cdn)
         self.expire_index()
         result = self.run_tt("echo")
         self.assertEqual(result.returncode, 3, result.stderr)
-        self.assertIn("cached", result.stderr)
 
     def test_tools_with_a_wrong_checksum_are_refused(self):
         self.write_manifest(self.work)
@@ -225,16 +233,19 @@ class TtTest(unittest.TestCase):
     def test_update_replaces_itself_and_installs_wrappers(self):
         tt_copy = os.path.join(self.root, "tt.py")
         shutil.copy(TT_PATH, tt_copy)
-        self.publish_tt({"tt.py": "# updated\n", "tt": "#!/bin/sh\n", "tt.cmd": "@echo off\r\n"})
+        with open(os.path.join(self.root, "tt.cmd"), "w") as file:
+            file.write("@echo off\r\n")
+        self.publish_tt({"tt.py": "# updated\n", "tt": "#!/bin/sh\n", "tt.ps1": "exit 0\r\n"})
         self.write_manifest(self.work)
         result = self.run_tt("update-self", tt_path=tt_copy)
         self.assertEqual(result.returncode, 0, result.stderr)
         with open(tt_copy) as file:
             self.assertEqual(file.read(), "# updated\n")
         self.assertTrue(os.access(os.path.join(self.root, "tt"), os.X_OK))
-        self.assertTrue(os.access(os.path.join(self.root, "tt.cmd"), os.X_OK))
+        self.assertTrue(os.access(os.path.join(self.root, "tt.ps1"), os.X_OK))
         self.assertTrue(os.access(tt_copy, os.X_OK))
-        self.assertEqual(sorted(os.listdir(self.root)), ["cdn", "home", "tt", "tt.cmd", "tt.py", "work"])
+        # The obsolete tt.cmd is removed
+        self.assertEqual(sorted(os.listdir(self.root)), ["cdn", "home", "tt", "tt.ps1", "tt.py", "work"])
 
     def test_update_rejects_invalid_python(self):
         tt_copy = os.path.join(self.root, "tt.py")

tt.cmd

@@ -1,9 +0,0 @@
-@echo off
-rem Runs tt.py from the directory of this script, on Windows
-where py >nul 2>nul
-if %ERRORLEVEL% equ 0 (
-    py -3 "%~dp0tt.py" %*
-) else (
-    python "%~dp0tt.py" %*
-)
-exit /b %ERRORLEVEL%

tt.ps1

@@ -0,0 +1,9 @@
+# Runs tt.py from the folder of this script, on Windows.
+# PowerShell passes the arguments to Python as they are, without parsing them again.
+$script = Join-Path $PSScriptRoot "tt.py"
+if (Get-Command py -ErrorAction SilentlyContinue) {
+    & py -3 $script @args
+} else {
+    & python $script @args
+}
+exit $LASTEXITCODE

tt.py

@@ -5,7 +5,7 @@
 "tt <command> [arguments]" runs <command>.py from the tools of that SDK version.
 When TACTILITY_SDK_PATH is set, its SDK and tools are used instead of downloading them.
 Both ~/.tactility and TACTILITY_SDK_PATH hold sdk/<version>/tools/ and sdk/<version>/platform-<platform>/TactilitySDK/.
-The tt (Linux, macOS) and tt.cmd (Windows) wrappers next to this script run it.
+The tt (Linux, macOS) and tt.ps1 (Windows PowerShell) wrappers next to this script run it.
 The SDK version is read from ./manifest.properties, or else from the first Apps/*/manifest.properties or apps/*/manifest.properties.
 """
 
@@ -20,7 +20,7 @@ import time
 import urllib.request
 import zipfile
 
-TT_VERSION = "1.0.0"
+TT_VERSION = "1.1.0"
 CDN_URL = os.environ.get("TT_CDN", "https://cdn.tactilityproject.org")
 HOME_DIRECTORY = os.path.join(os.path.expanduser("~"), ".tactility")
 INDEX_VALIDITY_SECONDS = 3600
@@ -143,16 +143,9 @@ def install_tools(version, version_directory, index):
         fail(f"SDK {version} has no tools")
     tools_directory = os.path.join(version_directory, "tools")
     hash_path = os.path.join(version_directory, "tools.sha256")
-    if os.path.isdir(tools_directory) and os.path.isfile(hash_path):
-        with open(hash_path) as file:
-            if file.read().strip() == tools["sha256"]:
-                return
     print(f"Downloading the tools of SDK {version}", file=sys.stderr)
     zip_path = os.path.join(version_directory, "tools.zip")
     if not download(sdk_url(version, tools["file"]), zip_path):
-        if os.path.isdir(tools_directory):
-            warn(f"using the cached tools of SDK {version}")
-            return
         fail(f"failed to download the tools of SDK {version}")
     if file_sha256(zip_path) != tools["sha256"]:
         os.remove(zip_path)
@@ -167,8 +160,6 @@ def install_platforms(version, version_directory, index, platforms):
     available = index.get("platforms", {})
     for platform in platforms:
         platform_directory = os.path.join(version_directory, f"platform-{platform}")
-        if os.path.isdir(os.path.join(platform_directory, "TactilitySDK")):
-            continue
         if platform not in available:
             fail(f"platform {platform} is not available in SDK {version}")
         print(f"Downloading SDK {version} for {platform}", file=sys.stderr)
@@ -183,11 +174,17 @@ def install_platforms(version, version_directory, index, platforms):
 
 
 def prepare_sdk(version, platforms):
+    """Installs what's missing of the SDK version. Installed tools and platforms are only replaced by "tt update-sdk", so they stay from the same build."""
     version_directory = version_path(HOME_DIRECTORY, version)
+    has_tools = os.path.isdir(os.path.join(version_directory, "tools"))
+    missing_platforms = [platform for platform in platforms if not os.path.isdir(os.path.join(version_directory, f"platform-{platform}", "TactilitySDK"))]
+    if has_tools and not missing_platforms:
+        return version_directory
     os.makedirs(version_directory, exist_ok=True)
     index = load_index(version, version_directory)
-    install_tools(version, version_directory, index)
-    install_platforms(version, version_directory, index, platforms)
+    if not has_tools:
+        install_tools(version, version_directory, index)
+    install_platforms(version, version_directory, index, missing_platforms)
     return version_directory