ENG-1646: self-heal scratchpad venv creation instead of retrying an identical failure

anton/core/backends/local.py

@@ -317,28 +317,80 @@ def _ensure_venv(self) -> None:
         if venv_path.is_dir():
             self._nuke_venv()
 
+        # ENG-1646: `uv venv` normally symlinks bin/python straight to the
+        # base interpreter, but on some hosts (observed reliably when this
+        # process is a descendant of a packaged/signed desktop app) it
+        # instead writes a freshly-copied, merely ad-hoc-signed launcher
+        # that's missing its own libpythonX.Y.dylib — macOS's AMFI then
+        # kills it outright ("Unrecoverable CT signature issue"), and every
+        # retry of the *same* uv invocation fails identically since it's not
+        # a transient condition. So retries escalate strategy instead of
+        # repeating the same doomed call: last attempt bypasses uv entirely
+        # and uses stdlib venv.create(symlinks=True), which has proven
+        # reliable in every environment we've seen this fail in.
         last_error: Exception | None = None
         for attempt in range(1, self._MAX_VENV_RETRIES + 1):
+            force_stdlib = attempt == self._MAX_VENV_RETRIES
             try:
-                self._create_venv()
+                self._create_venv(force_stdlib=force_stdlib)
                 if self._verify_venv_python():
                     self._setup_parent_site_packages()
                     self._save_python_version()
                     return
-                detail = f" ({self._last_verify_error})" if self._last_verify_error else ""
+                detail = self._diagnose_broken_interpreter()
                 raise RuntimeError(
                     f"venv Python binary at {self._venv_python} is not functional{detail}"
                 )
             except Exception as exc:
                 last_error = exc
-                self._nuke_venv()
+                # Keep the last attempt's directory around so it can be
+                # inspected instead of erasing the only evidence of what
+                # went wrong.
+                if attempt < self._MAX_VENV_RETRIES:
+                    self._nuke_venv()
 
         raise RuntimeError(
             f"Failed to create a working Python venv after {self._MAX_VENV_RETRIES} "
-            f"attempts. Last error: {last_error}. "
+            f"attempts (including a stdlib venv.create fallback). Last error: {last_error}. "
+            f"The broken venv was left at {venv_path} for inspection. "
             f"Try running: python3 -c 'print(\"ok\")' to verify your Python installation."
         )
 
+    def _diagnose_broken_interpreter(self) -> str:
+        """Best-effort detail for a failed `_verify_venv_python()`, beyond
+        the bare exit code/exception `_last_verify_error` already holds —
+        specifically, whether bin/python is a symlink or a copy, and its
+        code-signing status. Never raises; returns "" if nothing useful
+        could be gathered (e.g. the path doesn't exist at all)."""
+        base = f" ({self._last_verify_error})" if self._last_verify_error else ""
+        py = self._venv_python
+        if not py or not os.path.exists(py):
+            return base
+        try:
+            is_link = os.path.islink(py)
+            size = os.path.getsize(py)
+            extra = f"is_symlink={is_link}, size={size}"
+            if not is_link and sys.platform == "darwin":
+                import subprocess as _sp
+
+                try:
+                    result = _sp.run(
+                        ["codesign", "-dv", py],
+                        capture_output=True,
+                        timeout=5,
+                    )
+                    sig = (result.stderr or result.stdout or b"").decode(
+                        "utf-8", errors="replace"
+                    ).strip().splitlines()
+                    sig_line = next((l for l in sig if "flags=" in l or "Signature=" in l), "")
+                    if sig_line:
+                        extra += f", {sig_line}"
+                except Exception:
+                    pass
+            return f"{base} [{extra}]" if base else f" [{extra}]"
+        except OSError:
+            return base
+
     @staticmethod
     def _find_uv() -> str | None:
         uv = shutil.which("uv")
@@ -368,13 +420,13 @@ def _find_uv() -> str | None:
                 return candidate
         return None
 
-    def _create_venv(self) -> None:
+    def _create_venv(self, *, force_stdlib: bool = False) -> None:
         import subprocess as _sp
 
         self._venv_dir = str(self._venvs_base / self.name)
         os.makedirs(self._venv_dir, exist_ok=True)
 
-        uv = self._find_uv()
+        uv = None if force_stdlib else self._find_uv()
         if uv:
             try:
                 _sp.run(
@@ -417,6 +469,31 @@ def _create_venv(self) -> None:
         else:
             bin_dir = os.path.join(self._venv_dir, "bin")
             self._venv_python = os.path.join(bin_dir, "python")
+            if uv:
+                self._repair_copied_launcher(bin_dir)
+
+    def _repair_copied_launcher(self, bin_dir: str) -> None:
+        """ENG-1646: uv is supposed to symlink bin/python straight to the base
+        interpreter, but has been observed writing a freshly-copied, merely
+        ad-hoc-signed launcher instead — one macOS's AMFI then refuses to
+        execute, and which is also missing its own libpythonX.Y.dylib. A
+        plain symlink to the same interpreter has been reliable in every
+        case we've seen, so if uv didn't produce one, force it. Best-effort:
+        any failure here just leaves the (already broken) file in place for
+        `_verify_venv_python()` to catch downstream.
+        """
+        target = os.path.realpath(sys.executable)
+        for name in ("python", f"python{sys.version_info.major}", f"python{sys.version_info.major}.{sys.version_info.minor}"):
+            link_path = os.path.join(bin_dir, name)
+            try:
+                if os.path.islink(link_path):
+                    continue
+                if not os.path.exists(link_path):
+                    continue
+                os.remove(link_path)
+                os.symlink(target, link_path)
+            except OSError:
+                pass
 
     def venv_python(self) -> str | None:
         """Public accessor for the scratchpad's Python interpreter path.

tests/test_local_venv_provisioning.py

@@ -181,7 +181,7 @@ def fake_verify():
         pad._last_verify_error = "exit 1: dyld: Library not loaded"
         return False
 
-    monkeypatch.setattr(pad, "_create_venv", lambda: None)
+    monkeypatch.setattr(pad, "_create_venv", lambda **_: None)
     monkeypatch.setattr(pad, "_verify_venv_python", fake_verify)
 
     with pytest.raises(RuntimeError, match="dyld: Library not loaded"):
@@ -211,3 +211,135 @@ def test_find_uv_checks_winget_links_on_windows(monkeypatch):
     monkeypatch.setattr(local.os, "access", lambda p, mode: True)
 
     assert local.LocalScratchpadRuntime._find_uv() == winget_path
+
+
+# --- ENG-1646: uv sometimes writes bin/python as a copied, merely ad-hoc
+# signed launcher (missing its own libpythonX.Y.dylib) instead of a symlink
+# to the base interpreter. macOS's AMFI then refuses to execute it. A plain
+# symlink to the same interpreter has proven reliable everywhere we've seen
+# this fail, so `_create_venv()` now repairs the copy into a symlink, and
+# `_ensure_venv()`'s last retry escalates to the stdlib fallback instead of
+# repeating the same doomed `uv venv` call a third time.
+
+
+def _make_uv_venv_layout(tmp_path, venv_dir, *, python_is_symlink, real_interpreter=None):
+    """Lay out a bin/ dir the way `uv venv` would, with bin/python either a
+    symlink (the healthy/expected form) or a plain copied file (the observed
+    broken form)."""
+    bin_dir = os.path.join(venv_dir, "bin")
+    os.makedirs(bin_dir, exist_ok=True)
+    py = os.path.join(bin_dir, "python")
+    if python_is_symlink:
+        os.symlink(real_interpreter or sys.executable, py)
+    else:
+        with open(py, "wb") as f:
+            f.write(b"\xfa\xde\x0c\xfe fake copied launcher, not a real Mach-O")
+        os.chmod(py, 0o755)
+    return bin_dir
+
+
+def test_repair_copied_launcher_replaces_a_copy_with_a_symlink(tmp_path):
+    if sys.platform == "win32":
+        pytest.skip("posix-only: symlink semantics differ on Windows")
+    pad = make_pad(tmp_path)
+    bin_dir = _make_uv_venv_layout(tmp_path, str(tmp_path / "v"), python_is_symlink=False)
+
+    pad._repair_copied_launcher(bin_dir)
+
+    py = os.path.join(bin_dir, "python")
+    assert os.path.islink(py)
+    assert os.path.realpath(py) == os.path.realpath(sys.executable)
+
+
+def test_repair_copied_launcher_leaves_an_existing_symlink_alone(tmp_path):
+    if sys.platform == "win32":
+        pytest.skip("posix-only: symlink semantics differ on Windows")
+    pad = make_pad(tmp_path)
+    bin_dir = _make_uv_venv_layout(
+        tmp_path, str(tmp_path / "v"), python_is_symlink=True, real_interpreter="/some/other/python"
+    )
+
+    pad._repair_copied_launcher(bin_dir)
+
+    # Untouched — still points at whatever it originally symlinked to, not
+    # silently repointed at this process's own interpreter.
+    assert os.readlink(os.path.join(bin_dir, "python")) == "/some/other/python"
+
+
+def test_create_venv_repairs_a_copied_launcher_when_uv_is_used(tmp_path, monkeypatch):
+    if sys.platform == "win32":
+        pytest.skip("posix-only: symlink semantics differ on Windows")
+    import subprocess
+
+    pad = make_pad(tmp_path)
+    monkeypatch.setattr(LocalScratchpadRuntime, "_find_uv", staticmethod(lambda: "/fake/uv"))
+
+    # _create_venv sets self._venv_dir = self._venvs_base / self.name before
+    # invoking uv, so build the venv dir path the same way it does.
+    venv_dir = str(tmp_path / pad.name)
+
+    def fake_run(args, **kwargs):
+        # Simulate `uv venv` "succeeding" but leaving bin/python as a copy,
+        # matching what was actually observed via macOS's AMFI/XProtect logs.
+        _make_uv_venv_layout(tmp_path, venv_dir, python_is_symlink=False)
+        return subprocess.CompletedProcess(args, 0)
+
+    monkeypatch.setattr(subprocess, "run", fake_run)
+
+    pad._create_venv()
+
+    assert os.path.islink(pad._venv_python)
+    assert os.path.realpath(pad._venv_python) == os.path.realpath(sys.executable)
+
+
+def test_ensure_venv_escalates_to_stdlib_on_the_last_attempt_even_with_uv(tmp_path, monkeypatch):
+    # The failure this fix targets is deterministic per environment, not
+    # transient — repeating the identical `uv venv` call 3 times bought
+    # nothing. The last attempt must actually try something different.
+    pad = make_pad(tmp_path)
+    monkeypatch.setattr(LocalScratchpadRuntime, "_find_uv", staticmethod(lambda: "/fake/uv"))
+    calls = []
+
+    def fake_create(*, force_stdlib=False):
+        calls.append(force_stdlib)
+        pad._venv_python = str(tmp_path / "never-verifies")
+
+    monkeypatch.setattr(pad, "_create_venv", fake_create)
+    monkeypatch.setattr(pad, "_verify_venv_python", lambda: False)
+
+    with pytest.raises(RuntimeError):
+        pad._ensure_venv()
+
+    assert calls == [False, False, True]
+
+
+def test_ensure_venv_preserves_the_venv_dir_after_the_final_failure(tmp_path, monkeypatch):
+    # Nuking the directory after every attempt, including the last, erased
+    # the only evidence of what actually went wrong before anyone could look.
+    pad = make_pad(tmp_path)
+
+    def fake_create(*, force_stdlib=False):
+        pad._venv_dir = str(tmp_path / pad.name)
+        os.makedirs(pad._venv_dir, exist_ok=True)
+        pad._venv_python = str(tmp_path / "never-verifies")
+
+    monkeypatch.setattr(pad, "_create_venv", fake_create)
+    monkeypatch.setattr(pad, "_verify_venv_python", lambda: False)
+
+    with pytest.raises(RuntimeError):
+        pad._ensure_venv()
+
+    assert os.path.isdir(pad._venv_dir)
+
+
+def test_diagnose_broken_interpreter_reports_symlink_shape(tmp_path):
+    if sys.platform == "win32":
+        pytest.skip("posix-only: symlink semantics differ on Windows")
+    pad = make_pad(tmp_path)
+    copied = tmp_path / "copied_binary"
+    copied.write_bytes(b"not a real interpreter, just needs to exist")
+    pad._venv_python = str(copied)
+
+    detail = pad._diagnose_broken_interpreter()
+
+    assert "is_symlink=False" in detail