diff --git a/docs/NOW.md b/docs/NOW.md index 803e13475..c1d378de5 100644 --- a/docs/NOW.md +++ b/docs/NOW.md @@ -1,3 +1,16 @@ +# NOW -- a dead simulator is not a design that emitted nothing (2026-08-18) + +Last updated: 2026-08-18 + +## tools: check vvp's exit code and its own TIMEOUT marker (Closes #2193) + +- **Last site of the class, and it sits on the load-bearing proof.** `verify_emit_bitexact.py:127` took `vvp`'s stdout with no exit-code check. A crashed simulator yields a short Y-list, and the next check reports `step count RTL=0 PY=80` -- which reads as *the RTL emitted no outputs*, a design fault. Nothing had been compared +- **The testbench already said so and nobody read it.** It prints `TIMEOUT` into the very stdout the script parses. A design that ran out of simulated time is not a design that disagrees with the model, and the two want different fixes +- Exit code first (with the signal number when negative), then the `TIMEOUT` marker, then the step count -- which now states that vvp exited 0 and printed no TIMEOUT, so a mismatch there is real +- **Verified by shortening the testbench timeout** so the branch fires, not by reading. Clean tree still exits 0 +- **Class closed.** Two `run(...).stdout` sites remain, both `git ls-files` inside `try/except` with `check=True` and an rglob fallback: a git failure raises and is handled, not swallowed +- Dropped a hardcoded `#60000000` from the message on the second pass -- a literal copied into prose is exactly the drift this cycle has been correcting + # NOW -- two required checks were a single echo (2026-08-18) Last updated: 2026-08-18 diff --git a/tools/verify_emit_bitexact.py b/tools/verify_emit_bitexact.py index 376139817..6004b7e40 100644 --- a/tools/verify_emit_bitexact.py +++ b/tools/verify_emit_bitexact.py @@ -124,10 +124,30 @@ def check(g, arch, workdir): capture_output=True, text=True) if r.returncode != 0: print(f"FAIL {arch}: iverilog compile error\n{r.stderr}"); return False - out = subprocess.run(["vvp", vvp], capture_output=True, text=True).stdout + sim = subprocess.run(["vvp", vvp], capture_output=True, text=True) + out = sim.stdout + # A simulator that died, or one that hit the testbench's own TIMEOUT, produces a + # short Y-list -- which the step-count check below would report as "the RTL + # emitted the wrong number of outputs", i.e. a design fault. It is not: nothing + # was compared. Ask the exit code, and read the marker the testbench prints. + if sim.returncode != 0: + sig = f" (signal {-sim.returncode})" if sim.returncode < 0 else "" + print(f"FAIL {arch}: vvp exited {sim.returncode}{sig} -- the simulation did not " + f"run to completion, so nothing was compared") + for line in (sim.stderr or "").strip().splitlines()[:6]: + print(f" {line}") + return False + if "TIMEOUT" in out: + print(f"FAIL {arch}: the testbench hit its own TIMEOUT after " + f"{len(re.findall(r'^Y ', out, re.M))} of {len(py)} steps -- the design did " + f"not finish, which is not the same as disagreeing with the model") + return False rtl = [tuple(map(int, m.split())) for m in re.findall(r"^Y ([\d ]+)$", out, re.M)] if len(rtl) != len(py): - print(f"FAIL {arch}: step count RTL={len(rtl)} PY={len(py)}"); return False + print(f"FAIL {arch}: step count RTL={len(rtl)} PY={len(py)}" + f" -- vvp exited 0 and printed no TIMEOUT, so this is a real output-count " + f"disagreement, not a dead simulation") + return False mism = [(i, p, r_) for i, (p, r_) in enumerate(zip(py, rtl)) if p != r_] if mism: print(f"FAIL {arch}: {len(mism)}/{len(py)}; first step {mism[0][0]} py={mism[0][1]} rtl={mism[0][2]}")