Finish the diagnostic repair: a crash is not a numeric mismatch

docs/NOW.md

@@ -1,3 +1,16 @@
+# NOW -- finish the diagnostic repair; a crash is not a numeric mismatch (2026-08-18)
+
+Last updated: 2026-08-18
+
+## tools: 10 more sites took stdout without checking the exit code (Closes #2189)
+
+- **#2187 was a partial repair.** It fixed six `t27c gen-*` calls in one file; an AST re-scan found 12 more, in two classes
+- **Different call shape** (3 sites): the previous regex matched only the single-line form, so the multi-line `t27c gen-*` calls in `verify_multitarget.py` and `verify_trainer_c.py` were silently skipped
+- **Running the built binary** (8 sites): `.stdout` with no exit-code check means a crash arrives as a short or empty result list, and surfaces as a NUMERIC MISMATCH between targets. A program that died on signal 11 did not disagree about arithmetic -- and that is the most alarming reading the harness can produce
+- `_run_bin` joins `_build` and `_gen`, reporting the exit code and the signal when negative. All ten call sites now guard `None`, so a failure ends cleanly instead of raising `TypeError` on `None.split()`
+- **Verified by planting a fault:** `C target: cc exited 1` followed by `mod.h:18:34: error: use of undeclared identifier 'hm'`, where before there was only `FAIL: C backend failed to build/run`. All three tools still exit 0 on a clean tree
+- **Second partial repair in a row.** #2187's regex matched one call shape and I reported the pattern fixed without re-scanning. The AST scan is the check now, and it runs before the claim
+
 # NOW -- the wrappers threw the compiler's message away (2026-08-18)
 
 Last updated: 2026-08-18

tools/verify_igla_race.py

@@ -14,6 +14,25 @@
 """
 import os, re, sys, shutil, subprocess, tempfile, random
 
+def _run_bin(cmd, what, cwd=None):
+    """Run a built binary and return its stdout, or None with the reason printed.
+
+    Taking `.stdout` without checking the exit code means a crash arrives as a
+    short or empty result list, which then surfaces as a NUMERIC MISMATCH between
+    targets -- the most alarming reading available, and the wrong one. A program
+    that died on signal 11 did not disagree about arithmetic.
+    """
+    r = subprocess.run(cmd if isinstance(cmd, list) else [cmd],
+                       capture_output=True, text=True, cwd=cwd)
+    if r.returncode == 0:
+        return r.stdout
+    sig = f" (signal {-r.returncode})" if r.returncode < 0 else ""
+    print(f"  {what}: exited {r.returncode}{sig}")
+    for line in (r.stderr or "").strip().splitlines()[:4]:
+        print(f"      {line}")
+    return None
+
+
 def _gen(t27c, mode, spec, root):
     """Run `t27c gen-<mode>` and return its output, or None with the reason printed.
 
@@ -197,7 +216,9 @@ def run_c(t27c, vecs, wd):
     open(os.path.join(wd, "m.c"), "w").write(src)
     if not _build(["cc", "-O2", "-o", os.path.join(wd, "cb"), os.path.join(wd, "m.c")], wd, "core C"):
         return None
-    out = subprocess.run([os.path.join(wd, "cb")], capture_output=True, text=True).stdout
+    out = _run_bin(os.path.join(wd, "cb"), "core C run")
+    if out is None:
+        return None
     return [tuple(map(int, ln.split())) for ln in out.strip().splitlines()]
 
 
@@ -213,7 +234,9 @@ def run_rust(t27c, vecs, wd):
     rs = os.path.join(wd, "m.rs"); open(rs, "w").write(src)
     if not _build(["rustc", "-A", "warnings", "-O", "-o", os.path.join(wd, "rb"), rs], wd, "core Rust"):
         return None
-    out = subprocess.run([os.path.join(wd, "rb")], capture_output=True, text=True).stdout
+    out = _run_bin(os.path.join(wd, "rb"), "core Rust run")
+    if out is None:
+        return None
     return [tuple(map(int, ln.split())) for ln in out.strip().splitlines()]
 
 
@@ -251,7 +274,9 @@ def run_pe_c(t27c, vecs, wd):
     open(os.path.join(wd, "pe.c"), "w").write(src)
     if not _build(["cc", "-O2", "-o", os.path.join(wd, "peb"), os.path.join(wd, "pe.c")], wd, "systolic PE C"):
         return None
-    out = subprocess.run([os.path.join(wd, "peb")], capture_output=True, text=True).stdout
+    out = _run_bin(os.path.join(wd, "peb"), "systolic PE C run")
+    if out is None:
+        return None
     return [int(x) for x in out.split()]
 
 
@@ -279,7 +304,9 @@ def run_pe_rust(t27c, vecs, wd):
     rs = os.path.join(wd, "pe.rs"); open(rs, "w").write(src)
     if not _build(["rustc", "-A", "warnings", "-O", "-o", os.path.join(wd, "perb"), rs], wd, "systolic PE Rust"):
         return None
-    out = subprocess.run([os.path.join(wd, "perb")], capture_output=True, text=True).stdout
+    out = _run_bin(os.path.join(wd, "perb"), "systolic PE Rust run")
+    if out is None:
+        return None
     return [int(x) for x in out.split()]
 
 

tools/verify_multitarget.py

@@ -19,6 +19,42 @@
 """
 import os, sys, shutil, subprocess, tempfile, importlib.util, random
 
+def _run_bin(cmd, what, cwd=None):
+    """Run a built binary and return its stdout, or None with the reason printed.
+
+    Taking `.stdout` without checking the exit code means a crash arrives as a
+    short or empty result list, which then surfaces as a NUMERIC MISMATCH between
+    targets -- the most alarming reading available, and the wrong one. A program
+    that died on signal 11 did not disagree about arithmetic.
+    """
+    r = subprocess.run(cmd if isinstance(cmd, list) else [cmd],
+                       capture_output=True, text=True, cwd=cwd)
+    if r.returncode == 0:
+        return r.stdout
+    sig = f" (signal {-r.returncode})" if r.returncode < 0 else ""
+    print(f"  {what}: exited {r.returncode}{sig}")
+    for line in (r.stderr or "").strip().splitlines()[:4]:
+        print(f"      {line}")
+    return None
+
+
+def _gen(t27c, mode, spec, root):
+    """Run `t27c gen-<mode>` and return its output, or None with the reason printed.
+
+    Taking `.stdout` while checking neither the exit code nor stderr is how a spec
+    that failed to PARSE surfaced as "the C backend failed to build" for four days.
+    """
+    r = subprocess.run([t27c, "gen-" + mode, spec], capture_output=True, text=True, cwd=root)
+    if r.returncode == 0:
+        return r.stdout
+    out = (r.stderr or r.stdout or "").strip().splitlines()
+    print(f"  t27c gen-{mode} {spec}: exited {r.returncode}"
+          + ("" if out else " with no message"))
+    for line in out[:4]:
+        print(f"      {line}")
+    return None
+
+
 def _build(cmd, cwd, what):
     """Run a compiler and, if it fails, print what IT said before giving up.
 
@@ -88,8 +124,9 @@ def py_ref(g, fn, pairs):
 
 
 def run_c(t27c, spec, fn, pairs, wd):
-    hdr = subprocess.run([t27c, "gen-c", f"specs/ternary/{spec}.t27"],
-                         capture_output=True, text=True, cwd=ROOT).stdout
+    hdr = _gen(t27c, "c", f"specs/ternary/{spec}.t27", ROOT)
+    if hdr is None:
+        return None
     if "GFTSMUL_H" not in hdr and "GFTSADD_H" not in hdr:
         return None
     open(os.path.join(wd, "mod.h"), "w").write(hdr)
@@ -102,13 +139,16 @@ def run_c(t27c, spec, fn, pairs, wd):
     open(os.path.join(wd, "main.c"), "w").write(main)
     if not _build(["cc", "-O2", "-o", os.path.join(wd, "cbin"), os.path.join(wd, "main.c")], wd, "C target"):
         return None
-    out = subprocess.run([os.path.join(wd, "cbin")], capture_output=True, text=True).stdout
+    out = _run_bin(os.path.join(wd, "cbin"), "C target run")
+    if out is None:
+        return None
     return [int(x) for x in out.split()]
 
 
 def run_rust(t27c, spec, fn, pairs, wd):
-    src = subprocess.run([t27c, "gen-rust", f"specs/ternary/{spec}.t27"],
-                         capture_output=True, text=True, cwd=ROOT).stdout
+    src = _gen(t27c, "rust", f"specs/ternary/{spec}.t27", ROOT)
+    if src is None:
+        return None
     if "fn " not in src:
         return None
     a = ",".join(str(x) for x, _ in pairs); b = ",".join(str(y) for _, y in pairs)
@@ -117,7 +157,9 @@ def run_rust(t27c, spec, fn, pairs, wd):
     rs = os.path.join(wd, "m.rs"); open(rs, "w").write(src)
     if not _build(["rustc", "-A", "warnings", "-O", "-o", os.path.join(wd, "rbin"), rs], wd, "Rust target"):
         return None
-    out = subprocess.run([os.path.join(wd, "rbin")], capture_output=True, text=True).stdout
+    out = _run_bin(os.path.join(wd, "rbin"), "Rust target run")
+    if out is None:
+        return None
     return [int(x) for x in out.split()]
 
 

tools/verify_trainer_c.py

@@ -15,6 +15,42 @@
 """
 import os, re, sys, shutil, subprocess, tempfile, importlib.util, random
 
+def _run_bin(cmd, what, cwd=None):
+    """Run a built binary and return its stdout, or None with the reason printed.
+
+    Taking `.stdout` without checking the exit code means a crash arrives as a
+    short or empty result list, which then surfaces as a NUMERIC MISMATCH between
+    targets -- the most alarming reading available, and the wrong one. A program
+    that died on signal 11 did not disagree about arithmetic.
+    """
+    r = subprocess.run(cmd if isinstance(cmd, list) else [cmd],
+                       capture_output=True, text=True, cwd=cwd)
+    if r.returncode == 0:
+        return r.stdout
+    sig = f" (signal {-r.returncode})" if r.returncode < 0 else ""
+    print(f"  {what}: exited {r.returncode}{sig}")
+    for line in (r.stderr or "").strip().splitlines()[:4]:
+        print(f"      {line}")
+    return None
+
+
+def _gen(t27c, mode, spec, root):
+    """Run `t27c gen-<mode>` and return its output, or None with the reason printed.
+
+    Taking `.stdout` while checking neither the exit code nor stderr is how a spec
+    that failed to PARSE surfaced as "the C backend failed to build" for four days.
+    """
+    r = subprocess.run([t27c, "gen-" + mode, spec], capture_output=True, text=True, cwd=root)
+    if r.returncode == 0:
+        return r.stdout
+    out = (r.stderr or r.stdout or "").strip().splitlines()
+    print(f"  t27c gen-{mode} {spec}: exited {r.returncode}"
+          + ("" if out else " with no message"))
+    for line in out[:4]:
+        print(f"      {line}")
+    return None
+
+
 def _build(cmd, cwd, what):
     """Run a compiler and, if it fails, print what IT said before giving up.
 
@@ -113,8 +149,9 @@ def run_c(g, reg, steps, init_pairs, n_in, n_out, seq, t27c, wd):
     """Emit the trainer as a C program (t27c gen-c primitives + microcode interpreter
     + modf), compile, run the same sequence. Returns per-step output tuples, or None
     on a build failure. gftmod.h is (re)written into wd."""
-    hdr = subprocess.run([t27c, "gen-c", "specs/ternary/gft_smul.t27"],
-                         capture_output=True, text=True, cwd=ROOT).stdout
+    hdr = _gen(t27c, "c", "specs/ternary/gft_smul.t27", ROOT)
+    if hdr is None:
+        return None
     if "GFTSMUL_H" not in hdr:
         return None
     open(os.path.join(wd, "gftmod.h"), "w").write(hdr)
@@ -159,7 +196,9 @@ def run_c(g, reg, steps, init_pairs, n_in, n_out, seq, t27c, wd):
     b = os.path.join(wd, "tbin")
     if not _build(["cc", "-O2", "-o", b, cf], wd, "trainer C"):
         return None
-    out = subprocess.run([b], capture_output=True, text=True).stdout
+    out = _run_bin(b, "trainer C run")
+    if out is None:
+        return None
     return [tuple(map(int, ln.split())) for ln in out.strip().splitlines()]