Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .jules/bolt.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
## 2024-03-24 - AST Iteration Allocation Overhead
**Learning:** In the static analysis tools processing large ASTs recursively, eagerly materializing `ast.iter_child_nodes()` or `func_node.body` into lists (e.g., `list(ast.iter_child_nodes(node))`) introduces significant unnecessary memory allocation and garbage collection overhead. Since the traversal functions `_collect_return_paths` and `_assignment_callee` only iterate through the children exactly once, this eager list conversion is completely avoidable.
**Action:** Always accept `Iterable[ast.AST]` in recursive AST walker functions and pass the generator directly (`ast.iter_child_nodes()`) to avoid O(N) list allocation overhead at every node depth.
18 changes: 8 additions & 10 deletions src/wardline/scanner/taint/variable_level.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
from wardline.core.taints import _PROVENANCE_CLASH, RAW_ZONE, TRUST_RANK, TaintState, combine

if TYPE_CHECKING:
from collections.abc import Iterator
from collections.abc import Iterable, Iterator

# Serialisation sinks — calls that cross the representation boundary. Their
# output sheds validation provenance (raw bytes/str), so → UNKNOWN_RAW. This is
Expand Down Expand Up @@ -2414,7 +2414,7 @@ def compute_return_taint(
(the function's anchored *body* taint, pinned to its declaration).
"""
returns: list[tuple[TaintState, str | None, ast.expr]] = []
_collect_return_paths(list(func_node.body), function_taint, taint_map, var_taints, returns, return_snapshots)
_collect_return_paths(func_node.body, function_taint, taint_map, var_taints, returns, return_snapshots)
if not returns:
return None
result = returns[0][0]
Expand Down Expand Up @@ -2454,7 +2454,7 @@ def compute_return_callee(
beyond one hop stay ``None`` (the N-hop walk lives in the Loomweave stored-fact path).
"""
returns: list[tuple[TaintState, str | None, ast.expr]] = []
_collect_return_paths(list(func_node.body), function_taint, taint_map, var_taints, returns, return_snapshots)
_collect_return_paths(func_node.body, function_taint, taint_map, var_taints, returns, return_snapshots)
if not returns:
return None
worst = returns[0][0]
Expand All @@ -2468,14 +2468,14 @@ def compute_return_callee(
# a direct call. Provenance only — never changes a fire/no-fire decision.
for taint, callee, node in returns:
if taint == worst and callee is None and isinstance(node, ast.Name):
indirect = _assignment_callee(list(func_node.body), node.id, worst, function_taint, taint_map, var_taints)
indirect = _assignment_callee(func_node.body, node.id, worst, function_taint, taint_map, var_taints)
if indirect is not None:
return indirect
return None


def _assignment_callee(
nodes: list[ast.AST],
nodes: Iterable[ast.AST],
name: str,
worst: TaintState,
function_taint: TaintState,
Expand Down Expand Up @@ -2508,9 +2508,7 @@ def _assignment_callee(
and _resolve_expr(node.value, function_taint, taint_map, var_taints) == worst
):
result = callee
nested = _assignment_callee(
list(ast.iter_child_nodes(node)), name, worst, function_taint, taint_map, var_taints
)
nested = _assignment_callee(ast.iter_child_nodes(node), name, worst, function_taint, taint_map, var_taints)
if nested is not None:
result = nested
return result
Expand All @@ -2527,7 +2525,7 @@ def _return_callee(node: ast.expr) -> str | None:


def _collect_return_paths(
nodes: list[ast.AST],
nodes: Iterable[ast.AST],
function_taint: TaintState,
taint_map: dict[str, TaintState],
var_taints: dict[str, TaintState],
Expand Down Expand Up @@ -2558,7 +2556,7 @@ def _collect_return_paths(
taint = _resolve_expr(node.value, function_taint, taint_map, dict(snapshot or var_taints))
out.append((taint, _return_callee(node.value), node.value))
_collect_return_paths(
list(ast.iter_child_nodes(node)),
ast.iter_child_nodes(node),
function_taint,
taint_map,
var_taints,
Expand Down